大資料配置hadoop筆記
1.修改主機名
hostnamectl set-hostname master
bash
2.替換本地源
pkill -9 yum //殺掉yum程序
cd /etc/yum.repos.d
cat bigdata.repo //檢視本地源
rm -rf * //刪除
wget http://172.19.47.240/bigdata/repofile/bigdata.repo //下載本地源
cat bigdata.repo //檢視本地源
yum clean all //清除之前的yum快取
3.修改對映檔案
vi /etc/hosts
4.時間同步
tzselect
vim /etc/profile //將TZ='Asia/Shanghai'; export TZ貼上到裡面
source /etc/profile
date
5.下載ntp
yum install -y ntp
6.遮蔽預設server,設定master為本地時鐘源，伺服器層級設為10
vim /etc/ntp.conf //修改配置檔案
註釋掉
server 0.centos.pool.ntp.org.iburst
server 1.centos.pool.ntp.org.iburst
server 2.centos.pool.ntp.org.iburst
server 3.centos.pool.ntp.org.iburst
新增
server 127.127.1.0 #local clock
fudge 127.127.1.0 stratum 10
7.在早八-晚五時間段內每隔半個小時同步一次時間
systemctl status ntpd.service //看時鐘源是否開啟
systemctl start ntpd.service //開啟時鐘源服務
在slave01上同步master
ntpdate master
crontab -e //定時
*/30 8-17 * * * /usr/sbin/ntpdate master
8.配置免密登入
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa //生成金鑰
cat /root/.ssh/id_dsa.pub >> /root/.ssh/authorized_keys //將金鑰複製到authorized_keys
scp ~/.ssh/authorized_keys root@slave01:~/.ssh/ //遠端複製
9.安裝jdk
mdkir -p /usr/java
cd /usr/java
wget http://172.16.47.240/bigdata/bigdata_tar/jdk-8u171-linux-x64.tar.gz
tar -zxvf jdk-8u171-linux-x64.tar.gz -C /usr/java/
rm -rf /usr/java/jdk-8u171-linux-x64.tar.gz
vi /etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_171
export CLASSPATH=$JAVA_HOME/lib/
export PATH=$PATH:$JAVA_HOME/bin
source /etc/profile
10.安裝zookeeper
mkdir -p /usr/zookeeper && cd /usr/zookeeper
wget http://172.16.47.240/bigdata/bigdata_tar/zookeeper-3.4.10.tar.gz
tar -zxvf /usr/zookeeper/zookeeper-3.4.10.tar.gz -C /usr/zookeeper
rm -rf /usr/zookeeper/zookeeper-3.4.10.tar.gz
vi /etc/profile
export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10
export PATH=$PATH:$ZOOKEEPER_HOME/bin
source /etc/profile
cd /usr/zookeeper/zookeeper-3.4.10/conf && mv zoo_sample.cfg zoo.cfg && vi zoo.cfg
寫入
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdata
clientPort=2181
dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalog
server.1=master:2888:3888
server.2=slave01:2888:3888
server.3=slave02:2888:3888
cd zookeeper-3.4.10
mkdir zkdata zkdatalog
cd zkdata
vi myid
寫入
1
在slave01上
cd zookeeper-3.4.10
echo 2 >> myid
在slave02上
cd zookeeper-3.4.10
echo 3 >> myid
開啟
zkServer.sh start
zkServer.sh status
檢視錯誤資訊
cd zkdata
cat zookeeper.out
11.安裝hadoop
mkdir -p /usr/hadoop && cd /usr/hadoop
wget http://172.16.47.240/bigdata/bigdata_tar/hadoop-2.7.3.tar.gz
tar -zxvf /usr/hadoop/hadoop-2.7.3.tar.gz -C /usr/hadoop
rm -rf /sur/hadoop/hadoop-2.7.3.tar.gz
vi /etc/profile
export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
source /etc/profile
cd $HADOOP_HOME/etc/hadoop
vi hadoop-env.sh
vi core-site.xml
vi hdfs-site.xml
vi slaves
vi yarn-env.sh
echo "export JAVA_HOME=/usr/java/jdk1.8.0_171" >> yarn-env.sh
YARN框架的核心配置
vi yarn-site.xml
<!--指定ResourceManager的地址-->
<property>
<name>yarn.resourcemanager.address</name>
<value>master:18040</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:18030</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>master:18088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:18025</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>master:10141</value>
</property>
<!--指定reducer獲取資料的方式-->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
cp mapred-site.xml.template mapred-site.xml && vi mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
11.master為主節點，slave01和slave02為子節點
echo master > master
echo slave01 > slaves && echo slave2 >> slaves
12.格式化
cd hadoop-2.7.3
hadoop namenode -format
start-all.sh
13.安裝mysql
yum -y install mysql-community-server
systemctl start mysqld //開啟mysql
grep "temporary password" /var/log/mysqld.log //檢視密碼
mysql -u root -p
set global validate_password_policy=0; //降低密碼等級
set global validate_password_length=4;
alter user 'root'@'localhost' identified by '123456';
create user 'root'@'%' identified by '123456'; //建立使用者
grant all privileges on *.* to 'root'@'%' with grant option; //允許遠端連線
flush privileges; //重新整理許可權
create database hongyaa;
14.安裝hive
mkdir -p /usr/hive && cd /usr/hive
wget http://172.16.47.240/bigdata/bigdata_tar/apache-hive-2.1.1-bin.tar.gz
tar -zxvf /usr/hive/apache-hive-2.1.1-bin.tar.gz -C /usr/hive/
rm -rf /usr/hive/apache-hive-2.1.1-bin.tar.gz
vi /etc/profile
export HIVE_HOME=/usr/hive/apache-hive-2.3.6-bin
export PATH=$PATH:$HIVE_HOME/bin
source /etc/profile
配置hive執行環境
cd $HIVE_home/conf && vi hive-env.sh
# 配置Hadoop安裝路徑
export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3
# 配置Hive配置檔案存放路徑
export HIVE_CONF_DIR=/usr/hive/apache-hive-2.3.6-bin/conf
# 配置Hive執行資源路徑
export HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.3.6-bin/lib
解決版本衝突
cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib/
15.
cd $HIVE_HOME/lib && wget http://172.16.47.240/bigdata/bigdata_tar/mysql-connector-java-5.1.47-bin.jar
cd $HIVE_HOME/conf
vi hive-site.xml
寫入配置檔案
cd apache-hive-2.1.1-bin
schematool -dbType mysql -initSchema //資料初始化
hive --service metastore //開啟服務
轉為json
JSON.stringify(option)