Centos7 搭建 cloudera-manager 叢集
阿新 • • 發佈：2019-10-01
關閉防火牆
# 所有節點
systemctl stop firewalld
systemctl disable firewalld
禁止交換
# 所有節點
sysctl vm.swappiness=0
禁用大頁面
# 所有節點
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
1、JDK
解除安裝現有 Java
rpm -qa | grep java | xargs sudo rpm -e --nodeps
解壓到 /opt/module 目錄
tar -zxvf jdk-8u144-linux-x64.tar.gz -C /opt/module/
配置環境變數
sudo vim /etc/profile
#JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_144
export PATH=$PATH:$JAVA_HOME/bin
重新整理配置
source /etc/profile
測試是否安裝成功
java -version
2、MySQL
檢視是否現有的包
rpm -qa | grep mariadb
如果有，刪之
rpm -e --nodeps 包名
安裝 MariaDB
yum -y install mariadb mariadb-server
啟動 MariaDB
systemctl start mariadb
設定 MariaDB 自啟動
systemctl enable mariadb
進行 MariaDB 的相關簡單配置
mysql_secure_installation 首先是設定密碼，會提示先輸入密碼 Enter current password for root (enter for none):<–初次執行直接回車 設定密碼 Set root password? [Y/n] <– 是否設定root使用者密碼，輸入y並回車或直接回車 New password: <– 設定root使用者的密碼 Re-enter new password: <– 再輸入一次你設定的密碼 其他配置 Remove anonymous users? [Y/n] <– 是否刪除匿名使用者，回車 Disallow root login remotely? [Y/n] <–是否禁止root遠端登入,回車 Remove test database and access to it? [Y/n] <– 是否刪除test資料庫，回車 Reload privilege tables now? [Y/n] <– 是否重新載入許可權表，回車
配置遠端登入
mysql -uroot -p
MariaDB [(none)]> use mysql;
MariaDB [mysql]> update user set Host = '%' where Host = 'localhost' and User = 'root';
MariaDB [mysql]> flush privileges;
3、關閉 SELINUX
臨時關閉
setenforce 0
永久關閉
vim /etc/selinux/config
SELINUX=disabled
4、SSH 免密登入
生成公鑰和私鑰
[djm@hadoop102 .ssh]$ ssh-keygen -t rsa
[djm@hadoop103 .ssh]$ ssh-keygen -t rsa
[djm@hadoop104 .ssh]$ ssh-keygen -t rsa
將公鑰拷貝到要免密登入的目標機器上
[djm@hadoop102 .ssh]$ ssh-copy-id hadoop102
[djm@hadoop102 .ssh]$ ssh-copy-id hadoop103
[djm@hadoop102 .ssh]$ ssh-copy-id hadoop104
[djm@hadoop103 .ssh]$ ssh-copy-id hadoop102
[djm@hadoop103 .ssh]$ ssh-copy-id hadoop103
[djm@hadoop103 .ssh]$ ssh-copy-id hadoop104
[djm@hadoop104 .ssh]$ ssh-copy-id hadoop102
[djm@hadoop104 .ssh]$ ssh-copy-id hadoop103
[djm@hadoop104 .ssh]$ ssh-copy-id hadoop104
5、下載第三方依賴
# 所有節點
yum -y install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb httpd mod_ssl
6、建立 CM 用的資料庫
create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
7、解壓 cloudera-manager-centos7-cm5.16.2_x86_64.tar.gz
mkdir /opt/module/cloudera-manager
tar -zxvf /opt/software/cloudera-manager-centos7-cm5.16.2_x86_64.tar.gz -C /opt/module/cloudera-manager/
8、建立 cloudera-scm 使用者
# 所有節點
useradd --system --home=/opt/module/cloudera-manager/cm-5.16.2/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
--system 建立一個系統賬戶
--home 指定使用者登入時的主目錄，替換系統預設值/home/<使用者名稱>
--no-create-home 不要建立使用者的主目錄
--shell 使用者的登入 shell 名
--comment 使用者的描述資訊
9、配置 agent
vim /opt/module/cloudera-manager/cm-5.16.2/etc/cloudera-scm-agent/config.ini
# Hostname of the CM server.
server_host=hadoop102
10、初始化資料庫
建立使用者並授權
CREATE USER 'scm'@'%' IDENTIFIED BY 'scm';
grant all on *.* to 'scm'@'%' identified by 'scm' with grant option;
flush privileges;
拷貝 JDBC 驅動到目錄 /usr/share/java/
cp /opt/software/mysql-connector-java-5.1.27-bin.jar /usr/share/java/
mv /usr/share/java/mysql-connector-java-5.1.27-bin.jar /usr/share/java/mysql-connector-java.jar
建立 cm 資料庫
/opt/module/cloudera-manager/cm-5.16.2/share/cmf/schema/scm_prepare_database.sh mysql -h hadoop102 -uroot -p123456 --scm-host hadoop102 scm scm scm
-h：Database host
-u：Database username
-p：Database Password
--scm-host：SCM server's hostname
scm scm scm：database_name username password
11、建立 Parcel目錄
mkdir -p /opt/cloudera/parcel-repo
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
cp /opt/software/CDH* manifest.json /opt/cloudera/parcel-repo/
mv /opt/software/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha1 /opt/software/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha
# 所有節點
mkdir -p /opt/cloudera/parcels
chown cloudera-scm:cloudera-scm /opt/cloudera/parcels
12、分發
scp -r /opt/module/cloudera-manager/ hadoop103:/opt/module/
scp -r /opt/module/cloudera-manager/ hadoop104:/opt/module/
scp -r /opt/cloudera/parcel-repo/ hadoop103:/opt/cloudera/
scp -r /opt/cloudera/parcel-repo/ hadoop104:/opt/cloudera/
13、啟動和關閉
/opt/module/cloudera-manager/cm-5.16.2/etc/init.d/cloudera-scm-server start
/opt/module/cloudera-manager/cm-5.16.2/etc/init.d/cloudera-scm-server stop
# 所有節點
/opt/module/cloudera-manager/cm-5.16.2/etc/init.d/cloudera-scm-agent start
/opt/module/cloudera-manager/cm-5.16.2/etc/init.d/cloudera-scm