Ceph luminous 安裝配置

#環境centos7 , Ceph V12

openstack pike 與 ceph 集成 http://www.cnblogs.com/elvi/p/7897191.html

#環境準備
###########################
#升級內核(可選,)
#官方說明 http://docs.ceph.com/docs/master/start/os-recommendations/#linux-kernel
#CentOS7 升級內核到4.4 LTS

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
 

rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
#安裝kernel最新穩定版
yum --enablerepo=elrepo-kernel install kernel-lt-devel kernel-lt -y
awk -F\‘ ‘$1=="menuentry " {print $2}‘ /etc/grub2.cfg #查看啟動項
grub2-set-default 0 #設置第一行為默認啟動
rpm -e elrepo-release #刪除源
reboot
#uname -r #重啟後查看內核
#4.4
 
.98-1.el7.elrepo.x86_64
###########################


###########################

#使用阿裏源
rm -f /etc/yum.repos.d/*
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
sed -i ‘/aliyuncs.com/d‘ /etc/yum.repos.d/*.repo #刪除阿裏內網地址
#創建ceph源
echo ‘#阿裏ceph源
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/
gpgcheck=0
[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/
gpgcheck=0
[ceph-source]
name=ceph-source
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/
gpgcheck=0
#‘>/etc/yum.repos.d/ceph.repo
yum clean all && yum makecache #生成緩存

#關閉selinux、防火墻
systemctl stop firewalld.service
systemctl disable firewalld.service
firewall-cmd --state
sed -i ‘/^SELINUX=.*/c SELINUX=disabled‘ /etc/selinux/config
sed -i ‘s/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g‘ /etc/selinux/config
grep --color=auto ‘^SELINUX‘ /etc/selinux/config
setenforce 0

#時間同步
[[ -f /usr/sbin/ntpdate ]] || { echo "install ntp";yum install ntp -y &> /dev/null; } #若沒NTP則安裝
/usr/sbin/ntpdate ntp6.aliyun.com 
echo "*/3 * * * * /usr/sbin/ntpdate ntp6.aliyun.com  &> /dev/null" > /tmp/crontab
crontab /tmp/crontab

###########################
#網絡規劃

# #外網規劃
# 10.2.1.71 ceph01

#內網規劃，寫入hosts
echo ‘#ceph
192.168.58.71 ceph01
192.168.58.72 ceph02
192.168.58.73 ceph03
192.168.58.74 ceph04
‘>>/etc/hosts

#配置hostname、固定ip
Nu=1
hostnamectl set-hostname ceph0$Nu
NetName=ens37 #網卡名稱
rm -f /etc/sysconfig/network-scripts/ifcfg-$NetName
nmcli con add con-name $NetName ifname $NetName autoconnect yes type ethernet ip4 192.168.58.7$Nu/24  ipv4.dns "172.16.11.14 119.29.29.29" #ipv4.gateway "192.168.58.1"
nmcli connection reload #重新加載網絡


#磁盤(添加2快磁盤)
# [root@ceph01 ~]# lsblk
# NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT

# sdb      8:16   0  100G  0 disk 
# sdc      8:32   0  100G  0 disk

###########################

#ceph安裝配置
###########################
#全部在ceph01操作

# #Ceph節點簡介
# Mon(Montior)監控節點
# OSD(Object Storage Device)存儲節點
# MDS(Meta Data Server)，Ceph作文件系統用到，本實例不配置

#ceph01作為部署節點
#ceph01 ceph02 ceph03作為Mon
#ceph02 ceph03 ceph04作為OSD

#ssh免密驗證
#下載shell #
curl http://elven.vip/ks/sh/sshkey.me.sh >sshkey.me.sh
#認證用戶及密碼#
echo "
USER=root
PASS=123321
">my.sh.conf
sh ./ssh.sh ceph01 ceph02 ceph03 ceph04

#安裝ceph-deploy配置工具
yum install -y ceph-deploy

#創建配置目錄
mkdir /etc/ceph
cd /etc/ceph/

#初始化Mon配置
ceph-deploy new ceph{01,02,03}
#ceph-deploy new --cluster-network 192.168.58.0/24 --public-network 10.2.1.0/24 ceph{01,02,03}

# #修改冗余份數為2，日誌大小2G
# #配置網絡,單網卡忽略
echo ‘
public network = 10.2.1.0/24
cluster network = 192.168.58.0/24

mon_clock_drift_allowed = 2    
osd_journal_size = 4086
osd_pool_default_pg_num = 128
osd_pool_default_pgp_num = 128
osd pool default size = 2
osd pool default min size = 1
rbd_default_features = 1
client_quota = true
‘>>./ceph.conf

#安裝Ceph
ceph-deploy install ceph01 ceph02 ceph03 ceph04
#yum install -y ceph ceph-radosgw #實際上是安裝這2個rpm

#初始化monitor和key
cd /etc/ceph/
ceph-deploy --overwrite-conf mon create-initial

#創建存儲節點(使用sdb,sdc磁盤)
#lsblk #查看磁盤
#清空磁盤
ceph-deploy disk zap ceph02:sdb ceph02:sdc
ceph-deploy disk zap ceph03:sdb ceph03:sdc
ceph-deploy disk zap ceph04:sdb ceph04:sdc
#創建osd
ceph-deploy --overwrite-conf osd create ceph02:sdb ceph02:sdc
ceph-deploy --overwrite-conf osd create ceph03:sdb ceph03:sdc
ceph-deploy --overwrite-conf osd create ceph04:sdb ceph04:sdc

#拷貝配置及密鑰
ceph-deploy admin ceph01 ceph02 ceph03 ceph04
chmod 644 /etc/ceph/ceph.client.admin.keyring

#創建mon
ceph-deploy --overwrite-conf mon create ceph01
ceph-deploy --overwrite-conf admin ceph01
ceph-deploy --overwrite-conf mon create ceph02
ceph-deploy --overwrite-conf admin ceph02
ceph-deploy --overwrite-conf mon create ceph03
ceph-deploy --overwrite-conf admin ceph03

#添加mgr
#ceph 12開始，monitor必須添加mgr
ceph-deploy mgr create ceph{01,02,03}:mon_mgr
#ceph-deploy mgr create ceph01:mon_mgr ceph02:mon_mgr ceph03:mon_mgr

#啟用dashboard (在mon節點)
ceph mgr module enable dashboard
#http://ip:7000 訪問dashboard
netstat -antp|grep 7000

# 設置dashboard的ip和端口
# ceph config-key put mgr/dashboard/server_addr 192.168.68.71
# ceph config-key put mgr/dashboard/server_port 7000
# systemctl restart ceph-mgr@ceph01

###########################
#查看相關命令
ceph health
ceph -s
ceph osd tree

ceph df
ceph mon stat
ceph osd stat
ceph pg stat

ceph osd lspools
ceph auth list

###########################
#參考
https://segmentfault.com/a/1190000011581513
http://blog.csdn.net/wylfengyujiancheng/article/details/78461801

https://segmentfault.com/a/1190000011589519
https://www.cnblogs.com/netmouser/p/6876846.html

#中文 文檔
http://docs.ceph.org.cn/man/8/ceph-deploy/

#ceph工作原理和安裝
http://www.jianshu.com/p/25163032f57f

###########################

###########################
#簡單ceph性能測試

#創建測試池mytest
ceph osd pool create mytest 128
rados lspools
# ceph osd pool set mytest size 2 #副本為2
# ceph osd pool delete mytest #刪除


#Rados性能測試(關註 bandwidth帶寬,latency延遲)
rados bench -p mytest 10 write --no-cleanup #寫測試10秒
rados bench -p mytest 10 seq  #順序讀
rados bench -p mytest 10 rand #隨機讀
rados -p mytest cleanup #清理測試數據

#rbd塊設備測試
rbd create --size 2G mytest/test1 #創建塊設備映像test1
rbd ls mytest
rbd info mytest/test1

rbd map mytest/test1   #映射塊設備
#/dev/rbd0
#rbd showmapped         #查看已映射塊設備
#掛載
mkfs.xfs /dev/rbd0
mkdir -p /mnt/ceph
mount /dev/rbd0 /mnt/ceph/
df -h /mnt/ceph

#測試
rbd bench-write mytest/test1
#默認參數io 4k,線程數16,總寫入1024M, seq順序寫


rbd unmap mytest/test1 #取消塊設備映射
rbd rm mytest/test1    #刪除塊設備映像

#參考
https://www.cnblogs.com/sammyliu/p/5557666.html
###########################

###########################
#未解決問題：

##1、
#dashboard開啟後
#所有Mon開了7000端口，只能隨機1個Mon能訪問web


##2、
#rbd map映射塊設備失敗，暫沒解決
#環境 centos 7 內核升級了的 4.4.98-1.el7.elrepo.x86_64
#centos 7自帶內核3.10.0-693.2.2.el7.x86_64 正常

###########################

Ceph luminous 安裝配置