智警杯题解

题解"/>

智警杯题解

/

#配置hostname
#
#master
hostnamectl set-hostname master
#slave1
hostnamectl set-hostname slave1
#slave2
hostnamectl set-hostname slave2
#slave3
hostnamectl set-hostname slave3

#更换本地源
cd /etc/yum.repos.d/
rm -rf  *
wget http://172.16.47.240/bigdata/repofile/bigdata.repo
yum clean all
yum makecache

#master
cd /tmp
wget http://172.16.47.240/bigdata/bigdata_tar/jdk-8u171-linux-x64.tar.gz
wget http://172.16.47.240/bigdata/bigdata_tar/zookeeper-3.4.10.tar.gz
wget http://172.16.47.240/bigdata/bigdata_tar/hadoop-2.7.3.tar.gz
wget http://172.16.47.240/bigdata/bigdata_tar/apache-hive-2.1.1-bin.tar.gz

#slave1
cd /tmp
wget http://172.16.47.240/bigdata/bigdata_tar/mysql-connector-java-5.1.47-bin.jar

#关闭防火墙（all）
systemctl stop firewalld

#配置hosts(all)
echo '172.18.5.238 master master.root
172.18.5.241 slave1 slave1.root
172.18.5.242 slave2 slave2.root
172.18.5.240 slave3 slave3.root'>>/etc/hosts

#
echo '192.168.100.30 master master.root
192.168.100.31 slave1 slave1.root
192.168.100.32 slave2 slave2.root
192.168.100.33 slave3 slave3.root'>>/etc/hosts

#配置免密ssh(all)
# ssh-keygen -t rsa

rm -rf /root/.ssh/*
ssh-keygen -t rsa -P '' -f /root/.ssh/id_rsa
cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys

#master
ssh-copy-id master
ssh-copy-id slave1
ssh-copy-id slave2

ssh-copy-id slave3
#配置ntp(比赛时可先跳过，留到高可用前配置)

#时区设置（all）
timedatectl set-timezone Asia/Shanghai
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo "TZ='Asia/Shanghai'; export TZ;">> /etc/profile
source /etc/profile

#安装ntp服务(all)
yum -y install ntp
echo 'SYNC_HWCLOCK=yes'>>/etc/sysconfig/ntpd

#master
sed -i 's/^server/#&/'  /etc/ntp.conf
echo 'server 127.127.1.0
fudge 127.127.1.0 stratum 10
restrict 192.168.100.0 mask 255.255.255.0 nomodify notrap'>>/etc/ntp.conf
/bin/systemctl restart ntpd.service

#slave1和slave2和slave3
sed -i 's/^server/#&/'  /etc/ntp.conf
echo 'server 192.168.100.30
fudge 192.168.100.30 stratum 10
restrict 192.168.100.0 mask 255.255.255.0 nomodify notrap'>>/etc/ntp.conf
echo '*/10 * * * * /usr/sbin/ntpdate master'>/var/spool/cron/root
crontab -l
/bin/systemctl restart ntpd.service
ntpdate master

#all
ntpq -p

#配置环境变量(all)
echo '
#set java environment
export JAVA_HOME=/usr/java/jdk1.8.0_171
export PATH=$JAVA_HOME/bin:$PATH
#set zookeeper environment
export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10
export PATH=$PATH:$ZOOKEEPER_HOME/bin
#set hadoop enviroment
export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3/
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
'>>/etc/profile
source /etc/profile

#至此，可以先把xshell中的slave3会话先关了，防止误操作

#master
#配置jdk
mkdir /usr/java
tar -zxvf /tmp/jdk-8u171-linux-x64.tar.gz -C /usr/java/

scp -r /usr/java slave1:/usr/
scp -r /usr/java slave2:/usr/

scp -r /usr/java slave3:/usr/
#验证java配置(master slave1 slave2)
java -version

#配置zookeeper
#master

mkdir /usr/zookeeper
tar -zxvf /tmp/zookeeper-3.4.10.tar.gz -C /usr/zookeeper
mkdir -p /usr/zookeeper/zookeeper-3.4.10/zkdata
mkdir -p /usr/zookeeper/zookeeper-3.4.10/zkdatalog
echo 'tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdata
clientPort=2181
dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalog
server.1=master:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888'>/usr/zookeeper/zookeeper-3.4.10/conf/zoo.cfg
echo '1'>/usr/zookeeper/zookeeper-3.4.10/zkdata/myid

scp -r /usr/zookeeper slave1:/usr/
scp -r /usr/zookeeper slave2:/usr/

#slave1
echo '2'>/usr/zookeeper/zookeeper-3.4.10/zkdata/myid

#slave2
echo '3'>/usr/zookeeper/zookeeper-3.4.10/zkdata/myid

echo '4'>/usr/zookeeper/zookeeper-3.4.10/zkdata/myid
#master slave1 slave2
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh start
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh stop
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh status

#配置hadoop

mkdir -p /usr/hadoop
tar -zxvf /tmp/hadoop-2.7.3.tar.gz -C /usr/hadoop
echo 'export JAVA_HOME=/usr/java/jdk1.8.0_171'>>/usr/hadoop/hadoop-2.7.3/etc/hadoop/hadoop-env.sh
echo '<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
                                            <property>
                                                <name>fs.default.name</name>
                                                <value>hdfs://master:9000</value>
                                            </property>
                                            <property>
                                                <name>hadoop.tmp.dir</name>
                                                <value>/usr/hadoop/hadoop-2.7.3/hdfs/tmp</value>
                                            <description>A base for other temporary directories.</description>
                                            </property>
<property>
    <name>io.file.buffer.size</name>
    <value>131072</value>
</property>
                                            <property>
                                                <name>fs.checkpoint.period</name>
                                                <value>60</value>
                                            </property>
<property>
    <name>fs.checkpoint.size</name>
    <value>67108864</value>
</property>
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/core-site.xml
echo '<?xml version="1.0"?>
<configuration>
                                                                <property>
                                                                    <name>yarn.resourcemanager.address</name>
                                                                    <value>master:18040</value>
                                                                </property>
                                                                <property>
                                                                    <name>yarn.resourcemanager.scheduler.address</name>
                                                                    <value>master:18030</value>
                                                                </property>
                                                                <property>
                                                                    <name>yarn.resourcemanager.webapp.address</name>
                                                                    <value>master:18088</value>
                                                                </property>
                                                                <property>
                                                                    <name>yarn.resourcemanager.resource-tracker.address</name>
                                                                    <value>master:18025</value>
                                                                </property>
                                                                <property>
                                                                    <name>yarn.resourcemanager.admin.address</name>
                                                                    <value>master:18141</value>
                                                                </property>
                                                                <property>
                                                                    <name>yarn.nodemanager.aux-services</name>
                                                                    <value>mapreduce_shuffle</value>
                                                                </property>
<property>
    <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<!-- Site specific YARN configuration properties -->
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/yarn-site.xml
                                                                    echo 'slave1
                                                                    slave2'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/slaves
                                                                    echo 'master'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/master
echo '<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
                                                            <property>
                                                                <name>dfs.replication</name>
                                                                <value>3</value>
                                                            </property>
                                                            <property>
                                                                <name>dfs.namenode.name.dir</name>
                                                                <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/name</value>
                                                                <final>true</final>
                                                            </property>
                                                            <property>
                                                                <name>dfs.datanode.data.dir</name>
                                                                <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/data</value>
                                                                <final>true</final>
                                                            </property>
                                                            <property>
                                                                <name>dfs.namenode.secondary.http-address</name>
                                                                <value>master:9001</value>
                                                            </property>
                                                            <property>
                                                                <name>dfs.webhdfs.enabled</name>
                                                                <value>true</value>
                                                            </property>
<property>
    <name>dfs.permissions</name>
    <value>false</value>
</property>
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/hdfs-site.xml
cp /usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml.template /usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml
echo '<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
                                                                        <property>
                                                                            <name>mapreduce.framework.name</name>
                                                                            <value>yarn</value>
                                                                        </property>
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml
rm -rf /usr/hadoop/hadoop-2.7.3/share/doc

scp -r /usr/hadoop slave1:/usr/
scp -r /usr/hadoop slave2:/usr/

hadoop namenode -format
/usr/hadoop/hadoop-2.7.3/sbin/start-all.sh

jps

hdfs dfs -mkdir /college2020/

# #本地写好csv发到/tmp下
# hdfs dfs -put /tmp/student.csv /college2020/

#配置mysql

青椒

#slave2
#在线版本
cd /tmp
yum -y install mysql57-community-release-el7-10.noarch.rpm
yum -y install mysql-community-server
rpm -qa |grep mysql
mysqld --initialize
chown mysql:mysql -R /var/lib/mysql
systemctl start mysqld
systemctl enable mysqld
systemctl status mysqld
cat /var/log/mysqld.log |grep password
#vi /etc/myf
mysql -u root -p

alter user 'root'@'localhost' identified by '123456';
create user 'root'@'%' identified by '123456';
grant all privileges on *.* to 'root'@'%' identified by '123456';

create user 'qingjiao'@'%' identified by '123456';
alter user 'qingjiao'@'%' identified by '123456';
grant all privileges on *.* to 'qingjiao'@'%' identified by '123456';
flush privileges;

INSTALL PLUGIN validate_password SONAME 'validate_password.so';
set global validate_password_policy = 0;
set global validate_password_length = 4;
flush privileges;

quit;

mysql -u qingjiao -p

create database qingjiao;

quit;

#配置hive

#master
mkdir /usr/hive
tar -zxvf /tmp/apache-hive-2.1.1-bin.tar.gz -C /usr/hive/
scp -r /usr/hive slave1:/usr
echo 'export HIVE_HOME=/usr/hive/apache-hive-2.1.1-bin
export PATH=$PATH:$HIVE_HOME/bin'>>/etc/profile
source /etc/profile

cp /usr/hive/apache-hive-2.1.1-bin/lib/jline-2.12.jar /usr/hadoop/hadoop-2.7.3/share/hadoop/yarn/lib/
cp /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh.template /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh

echo '#Set HADOOP_HOME to point to a specific hadoop install directory
HADOOP_HOME=/usr/hadoop/hadoop-2.7.3
# Hive Configuration Directory can be controlled by:
export HIVE_CONF_DIR=/usr/hive/apache-hive-2.1.1-bin/conf' > /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh

echo '<configuration>
<property>
    <name>hive.metastore.warehouse.dir</name>
    <value>/usr/hive/hive_remote/warehouse</value>
</property>
<property>
    <name>hive.metastore.local</name>
    <value>false</value>
</property>
<property>
    <name>hive.metastore.uris</name>
    <value>thrift://slave1:9083</value>
</property>
</configuration>'>/usr/hive/apache-hive-2.1.1-bin/conf/hive-site.xml

#slave1

echo 'export HIVE_HOME=/usr/hive/apache-hive-2.1.1-bin
export PATH=$PATH:$HIVE_HOME/bin'>>/etc/profile
source /etc/profile

cp /tmp/mysql-connector-java-5.1.47-bin.jar /usr/hive/apache-hive-2.1.1-bin/lib

cp /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh.template /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh

echo '#Set HADOOP_HOME to point to a specific hadoop install directory
HADOOP_HOME=/usr/hadoop/hadoop-2.7.3
# Hive Configuration Directory can be controlled by:
export HIVE_CONF_DIR=/usr/hive/apache-hive-2.1.1-bin/conf' > /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh
cp /usr/hive/apache-hive-2.1.1-bin/conf/hive-default.xml.template /usr/hive/apache-hive-2.1.1-bin/conf/hive-site.xml
echo '<configuration>
#设置元数据位置
<property>
    <name>hive.metastore.warehouse.dir</name>
    <value>/usr/hive/hive_remote/warehouse</value>
</property>
#使用本地服务连接hive
<property>
    <name>hive.metastore.local</name>
    <value>true</value>
    <description>Use false if a production metastore server is used</description>
</property>
#配置连接Mysql，设置ssl连接方式为false或true
<property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
</property>
#设置Mysql驱动
<property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
</property>
#设置连接用户名
<property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>qingjiao</value>
</property>
#设置连接密码
<property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>123456</value>
</property>
#关闭metastore的schema一致性
<property>
    <name>hive.metastore.schema.verification</name>
    <value>false</value>
</property>
<property>
    <name>datanucleus.schema.autoCreateAll</name>
    <value>true</value>
</property>
<property>
    <name>hive.exec.scratchdir</name>
    <value>/usr/hive/hive-2.1.1/tmp</value>
</property>
<property>
    <name>hive.querylog.location</name>
    <value>/usr/hive/hive-2.1.1/logs</value>
</property>
</configuration>'>/usr/hive/apache-hive-2.1.1-bin/conf/hive-site.xml

#slave1
schematool -dbType mysql -initSchema
hive --service metastore

#master
hive

#验证hive
show databases;
create database student;
use student;
#验证HDFS
create table student(stu char(20),name char(20)) row format delimited fields terminated by ',';
load data inpath '/college2020/student.csv' into table student;
select * from student;
#验证YARN
create table sxy(stu char(8));
insert into sxy values(5);
select * from sxy;

#HA之前相关配置

#配置ssh
#master
ssh-copy-id slave3

#slave3
ssh-copy-id master

#slave1
ssh-copy-id slave2

#slave2
ssh-copy-id master
ssh-copy-id slave1

#配置slave3的jdk
#slave3
scp -r master:/usr/java /usr
java -version

#zookeeper
#slave3
scp -r master:/usr/zookeeper /usr/
echo '4'>/usr/zookeeper/zookeeper-3.4.10/zkdata/myid

#all
echo 'tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdata
clientPort=2181
dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalog
peerType=observer
server.1=master:2888:3888
server.2=slave1:2888:3888:observer
server.3=slave2:2888:3888
server.4=slave3:2888:3888'>/usr/zookeeper/zookeeper-3.4.10/conf/zoo.cfg

/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh stop
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh start
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh status

#hadoop HA

#master
stop-all.sh

#all
rm -rf /usr/hadoop

#master
mkdir -p /usr/hadoop
tar -zxvf /tmp/hadoop-2.7.3.tar.gz -C /usr/hadoop

echo 'export JAVA_HOME=/usr/java/jdk1.8.0_171'>>/usr/hadoop/hadoop-2.7.3/etc/hadoop/hadoop-env.sh

echo '<?xml version="1.0"?>
<configuration>
<property>
  <!--指定hdfs的方式的nameservice为qingjiao-->
  <name>fs.defaultFS</name>
   <value>hdfs://qingjiao</value>
</property>
<property>
  <!--指定hadoop工作目录-->
  <name>hadoop.tmp.dir</name>
   <value>/usr/hadoop/hadoop-2.7.3/hdfs/tmp</value>
</property>
<property>
  <!--指定zookeeper集群访问地址-->
  <name>ha.zookeeper.quorum</name>
   <value>master:2181,slave2:2181,slave3:2181</value>
</property>
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/core-site.xml

echo '<?xml version="1.0"?>
<configuration>
<property>
    <name>fs.defaultFS</name>
    <value>hdfs://qingjiao</value>
</property>
<property>
    <name>hadoop.tmp.dir</name>
    <value>/usr/hadoop/hadoop-2.7.3/hdfs/tmp</value>
</property>
<property>
    <name>ha.zookeeper.quorum</name>
    <value>master:2181,slave1:2181,slave2:2181,slave3:2181</value>
</property>
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/core-site.xml

echo 'slave1
slave2
slave3'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/slaves

echo '<?xml version="1.0"?>
<configuration>
 <property>
  <!--配置副本数-->
   <name>dfs.replication</name>
   <value>3</value>
 </property>
 <property>
   <!--指定hdfs的nameservice为qingjiao 需要和core-site保持一致-->
   <name>dfs.nameservices</name>
   <value>qingjiao</value>
 </property>
 <property>
   <!--指定services name里面设定NN列表-->
   <name>dfs.ha.namenodes.qingjiao</name>
   <value>nn1,nn2</value>
 </property>
 <property>
   <!--设置nn1的RPC通信地址和端口号 -rpc用于和DN通讯-->
   <name>dfs.namenode.rpc-address.qingjiao.nn1</name>
   <value>master:9000</value>
  </property>
  <property>
  <!--设置nn1的http通信地址和端口号 -用来和web客户端通讯-->
  <name>dfs.namenode.http-address.qingjiao.nn1</name>
   <value>master:50070</value>
  </property>
  <property>
  <!--设置nn2的RPC通信地址和端口号 -rpc用于和DN通讯-->
  <name>dfs.namenode.rpc-address.qingjiao.nn2</name>
   <value>slave3:9000</value>
  </property>
  <property>
  <!--设置nn2的http通信地址和端口号 -用来和web客户端通讯-->
  <name>dfs.namenode.http-address.qingjiao.nn2</name>
   <value>slave3:50070</value>
  </property>
  <property>
  <!--指定namenode间用于共享编辑日志的journal节点列表 -hdfs元数据存储位置-->
  <name>dfs.namenode.shared.edits.dir</name>
   <value>qjournal://master:8485;slave2:8485;slave3:8485/qingjiao</value>
  </property>
  <property>
  <!--指定journalnode 上用于存放edits日志的目录 -本地磁盘数据存储位置 自行设定-->
  <name>dfs.journalnode.edits.dir</name>
   <value>/usr/hadoop/hadoop-2.7.3/journaldata</value>
  </property>
  <property>
  <!--开启namenode失败自动切换 -->
  <name>dfs.ha.automatic-failover.enabled</name>
   <value>true</value>
  </property>
  <property>
  <!--配置失败自动切换实现方式 无需换行 实现类-->
  <name>dfs.client.failover.proxy.provider.qingjiao</name>
   <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
  </property>
  <property>
  <!--配置隔离机制，多个机制换行分割，防止脑裂/假死-->
  <name>dfs.ha.fencing.methods</name>
   <value>
    sshfence
  　</value>
  </property>
  <property>
  <!--使用 sshfenns离机制时需要 ssh 免登陆 RSA类型-->
  <name>dfs.ha.fencing.ssh.private-key-files</name>
   <value>/root/.ssh/id_rsa</value>
  </property>
  <property>
  <!--指定 sshfenns离机制超时时间-->
  <name>dfs.ha.fencing.ssh.connect-timeout</name>
   <value>3000</value>
  </property>
<property>
   <name>dfs.namenode.name.dir</name>
   <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/name</value>
   <final>true</final>
</property>
 <property>
   <name>dfs.datanode.data.dir</name>
   <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/data</value>
   <final>true</final>
 </property>
 <property>
   <name>dfs.webhdfs.enabled</name>
   <value>true</value>
 </property>
 </configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/hdfs-site.xml

echo '<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
    <name>dfs.nameservices</name>
    <value>qingjiao</value>
</property>
<property>
    <name>dfs.ha.namenodes.qingjiao</name>
    <value>nn1,nn2</value>
</property>
<property>
    <name>dfs.namenode.rpc-address.qingjiao.nn1</name>
    <value>master:8020</value>
</property>
<property>
    <name>dfs.namenode.rpc-address.qingjiao.nn2</name>
    <value>slave3:8020</value>
</property>
<property>
    <name>dfs.namenode.http-address.qingjiao.nn1</name>
    <value>master:50070</value>
</property>
<property>
    <name>dfs.namenode.http-address.qingjiao.nn2</name>
    <value>slave3:50070</value>
</property>
<property>
    <name>dfs.namenode.shared.edits.dir</name>
    <value>qjournal://master:8485;slave2:8485;slave3:8485/qingjiao</value>
</property>
<property>
    <name>dfs.journalnode.edits.dir</name>
    <value>/usr/hadoop/hadoop-2.7.3/tmp/data/dfs/jn</value>
</property>
<property>
    <name>dfs.client.failover.proxy.provider.qingjiao</name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
    <name>dfs.ha.fencing.methods</name>
    <value>sshfence</value>
    <value>shell(/bin/true)</value>
</property>
<property>
    <name>dfs.ha.fencing.ssh.private-key-files</name>
    <value>/root/.ssh/id_rsa</value>
</property>
<property>
    <name>dfs.ha.automatic-failover.enabled</name>
    <value>true</value>
</property>
<property>
    <name>dfs.replication</name>
    <value>3</value>
</property>
<property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/name</value>
    <final>true</final>
</property>
<property>
    <name>dfs.datanode.data.dir</name>
    <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/data</value>
    <final>true</final>
</property>
<property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>master:9001</value>
</property>
<property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
</property>
<property>
    <name>dfs.permissions</name>
    <value>false</value>
</property>
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/hdfs-site.xml

echo '<?xml version="1.0"?>
<configuration>
<property>
    <name>yarn.resourcemanager.ha.enabled</name>
    <value>true</value>
</property>
<property>
    <name>yarn.resourcemanager.cluster-id</name>
    <value>hongya</value>
</property>
<property>
    <name>yarn.resourcemanager.ha.rm-ids</name>
    <value>rm1,rm2</value>
</property>
<property> 
    <name>yarn.resourcemanager.ha.automatic-failover.enabled</name> 
    <value>true</value> 
</property>
<property>
    <name>yarn.resourcemanager.hostname.rm1</name>
    <value>slave1</value>
</property>
<property>
    <name>yarn.resourcemanager.hostname.rm2</name>
    <value>slave2</value>
</property>
<property>
    <name>yarn.resourcemanager.recovery.enabled</name> 
    <value>true</value> 
</property>
<property>
    <name>yarn.resourcemanager.zk-address</name>
    <value>master:2181,slave1:2181,slave2:2181</value>
</property>
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
<property>
    <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<!-- Site specific YARN configuration properties -->
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/yarn-site.xml

cp /usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml.template /usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml
echo '<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>
</configuration>'>/usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml

rm -rf /usr/hadoop/hadoop-2.7.3/share/docs

scp -r /usr/hadoop slave1:/usr/
scp -r /usr/hadoop slave2:/usr/
scp -r /usr/hadoop slave3:/usr/

#all
hadoop-daemon.sh stop journalnode
hadoop-daemon.sh start journalnode

#master
hadoop namenode -format
hadoop-daemon.sh start namenode
hadoop-daemon.sh start datanode

#slave3
hadoop namenode -bootstrapStandby

#master或slave3
/usr/hadoop/hadoop-2.7.3/bin/hdfs zkfc -formatZK

#master(注意哪一台namenode是active)
start-all.sh

#slave1和slave2
yarn-daemon.sh start resourcemanager
yarn-daemon.sh start nodemanager

# #mysql
# #离线版本
# groupadd mysql
# useradd -r -g mysql mysql

# cd /tmp
# tar xzvf mysql-5.7.24-linux-glibc2.12-x86_64.tar.gz
# mv mysql-5.7.24-linux-glibc2.12-x86_64 /usr/local/
# cd /usr/local/
# mv mysql-5.7.24-linux-glibc2.12-x86_64 mysql
# mkdir /usr/local/mysql/data

# chown -R mysql:mysql /usr/local/mysql
# chmod -R 755 /usr/local/mysql
# cd /usr/local/mysql/bin

# ./mysqld --initialize-insecure --user=mysql --datadir=/usr/local/mysql/data --basedir=/usr/local/mysql

# echo '[mysqld]
# datadir=/usr/local/mysql/data
# port = 3306
# sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
# symbolic-links=0
# max_connections=400
# innodb_file_per_table=1
# lower_case_table_names=1
# [mysqld_safe]
# log-error=/var/log/mariadb/mariadb.log
# pid-file=/var/run/mariadb/mariadb.pid' > /etc/myf
# mkdir /var/log/mariadb
# touch /var/log/mariadb/mariadb.log
# chown -R mysql:mysql /var/log/mariadb/

# /usr/local/mysql/support-files/mysql.server start
# ln -s /usr/local/mysql/support-files/mysql.server /etc/init.d/mysql 
# ln -s /usr/local/mysql/bin/mysql /usr/bin/mysql

# mysql

# set global validate_password_policy = 0;
# set global validate_password_length = 4;
# alter user 'root'@'localhost' identified by '123456';
# create user 'root'@'%' identified by '123456';
# grant all privileges on *.* to 'root'@'%' identified by '123456';
# flush privileges;