Ceph在CentOS上安裝單機客戶端

環境描述

Linux 版本：CentOS Linux release 7.2 (Final)
CEPH版本：ceph version 11.2.0

內網無法連線到官網地址，因此可以通過兩種方式進行安裝
A）部署內部YUM源站點，通過ceph-deploy工具指定URL
B）下載RPM包並上傳到伺服器，手工安裝
這裡採用第二種安裝方式

RPM依賴安裝

1、通過官方網站下載最新版RPM包
2、解決依賴關係
安裝之前，先檢查下列包是否存在，不同的Linux核心版本會有一些差異

yum install -y junit boost gdisk fcgi xmlstarlet java-devel snappy cryptsetup
yum install
 
 python-flask-0.10.1-4.el7.noarch
yum install gperftools-libs-2.2.1-1.2.x86_64.rpm
yum install libbabeltrace-1.2.4-3.el7.x86_64.rpm
yum install lttng-ust-2.4.1-1.el7.1.x86_64.rpm
yum install userspace-rcu-0.7.16-1.el7.x86_64.rpm
yum install python-requests
yum install python-setuptools
yum install glibc.i686
 

安裝準備

3、Jewel版本之後預設使用者為ceph，安裝之前先建立ceph使用者並授權

adduser -d /home/ceph -m ceph
passwd ceph
# 設定使用者許可權
echo "ceph ALL =(root) NOPASSWD:ALL" | tee /etc/sudoers.d/ceph
chmod 0440 /etc/sudoers.d/ceph

4、requiretty準備

visudo
# 註釋掉Defaults requiretty 
# Defaults requiretty修改為 #Defaults requiretty， 表示不需要控制終端。
# 否則會出現sudo: sorry, you must have a tty to run sudo
 

5、防火牆設定
根據公司安全策略決定是否關閉防火牆，如果不關閉，多臺機器間的通訊可能會出現埠不通的情況

systemctl stop firewalld.service
systemctl disable firewalld.service
setenforce 0

6、修改別名
ceph.conf中可以通過配置別名進行多個監控服務之間的通訊，修改hosts中的別名，或者通過hostnamectl設定

vi /etc/hosts
# xxx.xxx.xxx.xxx ceph-node1

安裝CEPH RPM包

rpm -ivh libbabeltrace-1.2.1-1.fc20.x86_64.rpm
rpm -ivh libbabeltrace-1.2.1-1.fc20.x86_64.rpm
rpm -ivh leveldb-1.12.0-5.el7.1.x86_64.rpm
rpm -ivh leveldb-devel-1.12.0-5.el7.1.x86_64.rpm 
rpm -ivh selinux-policy-3.13.1-60.el7_2.9.noarch.rpm
rpm -ivh selinux-policy-targeted-3.13.1-60.el7_2.9.noarch.rpm
# 下載並解壓CEPH安裝包
rpm -ivh *
#  1:librados2-1:11.1.0-6896.g2aab144.################################# [  3%]
#  2:librbd1-1:11.1.0-6896.g2aab144.el################################# [  6%]
#  3:libcephfs2-1:11.1.0-6896.g2aab144################################# [  9%]
#  4:python-rados-1:11.1.0-6896.g2aab1################################# [ 13%]
#  5:librados-devel-1:11.1.0-6896.g2aa################################# [ 16%]
#  6:librgw2-1:11.1.0-6896.g2aab144.el################################# [ 19%]
#  7:libradosstriper1-1:11.1.0-6896.g2################################# [ 22%]
#  8:python-rgw-1:11.1.0-6896.g2aab144################################# [ 25%]
#  9:python-cephfs-1:11.1.0-6896.g2aab################################# [ 28%]
# 10:python-rbd-1:11.1.0-6896.g2aab144################################# [ 31%]
# 11:ceph-common-1:11.1.0-6896.g2aab14################################# [ 34%]
# 12:ceph-selinux-1:11.1.0-6896.g2aab1################################# [ 38%]
# 13:ceph-base-1:11.1.0-6896.g2aab144.################################# [ 41%]
# 14:libcephfs_jni1-1:11.1.0-6896.g2aa################################# [ 44%]
# 15:ceph-mds-1:11.1.0-6896.g2aab144.e################################# [ 47%]
# 16:ceph-mgr-1:11.1.0-6896.g2aab144.e################################# [ 50%]
# 17:ceph-mon-1:11.1.0-6896.g2aab144.e################################# [ 53%]
# 18:ceph-osd-1:11.1.0-6896.g2aab144.e################################# [ 56%]
# 19:ceph-1:11.1.0-6896.g2aab144.el7  ################################# [ 59%]
# 20:cephfs-java-1:11.1.0-6896.g2aab14################################# [ 63%]
# 21:libcephfs_jni-devel-1:11.1.0-6896################################# [ 66%]
# 22:ceph-radosgw-1:11.1.0-6896.g2aab1################################# [ 69%]
# 23:ceph-test-1:11.1.0-6896.g2aab144.################################# [ 72%]
# 24:rbd-mirror-1:11.1.0-6896.g2aab144################################# [ 75%]
# 25:python-ceph-compat-1:11.1.0-6896.################################# [ 78%]
# 26:libradosstriper-devel-1:11.1.0-68################################# [ 81%]
# 27:librgw-devel-1:11.1.0-6896.g2aab1################################# [ 84%]
# 28:libcephfs-devel-1:11.1.0-6896.g2a################################# [ 88%]
# 29:librbd-devel-1:11.1.0-6896.g2aab1################################# [ 91%]
# 30:rbd-fuse-1:11.1.0-6896.g2aab144.e################################# [ 94%]
# 31:rbd-nbd-1:11.1.0-6896.g2aab144.el################################# [ 97%]
# 32:ceph-fuse-1:11.1.0-6896.g2aab144.################################# [100%]

配置ceph.conf檔案

ceph --version
# ceph version 11.1.0-6896-g2aab144 (2aab1443fa28e47e81add9a5aa999c5f7dc39e70)
# 將服務端的配置拷貝到客戶段
cd /etc/ceph
vi ceph.client.admin.keyring

[client.admin]
        key = AQDeBDFZCNYwDxAAwyhHxRK6Cd7OMDm==
        auid = 0
        caps mds = "allow"
        caps mon = "allow *"
        caps osd = "allow *"

vi ceph.conf

#新增以下內容
fsid = deeade81-2e18-4d1e-a37a-1154
mon_initial_members = ser-node1,ser-node2,ser-node3
mon_host = xxx.xxx.xxx,xxx.xxx.xxx,xxx.xxx.xxx
auth_cluster_required = none
auth_service_required = none
auth_client_required = none

mon_allow_pool_delete = true
mon clock drift allowed = 2
mon clock drift warn backoff = 30

#enable_experimental_unrecoverable_data_corrupting_features = ms-type-async
ms_type = async
ms_async_op_threads = 4

osd_find_best_info_ignore_history_les = true
mon_pg_warn_max_per_osd = 1000
max_open_files = 1000000
throttler_perf_counter = false
debug_lockdep = 0/0
debug_context = 0/0
debug_crush = 0/0
debug_mds = 0/0
debug_mds_balancer = 0/0
debug_mds_locker = 0/0
debug_mds_log = 0/0
debug_mds_log_expire = 0/0
debug_mds_migrator = 0/0
debug_buffer = 0/0
debug_timer = 0/0
debug_filer = 0/0
debug_striper = 0/0
debug_objecter = 0/0
debug_rados = 0/0
debug_rbd = 0/0
debug_rbd_mirror = 0/0
debug_rbd_replay = 0/0
debug_journaler = 0/0
debug_objectcacher = 0/0
debug_client = 0/0
debug_osd = 0/0
debug_optracker = 0/0
debug_objclass = 0/0
debug_filestore = 0/0
debug_journal = 0/0
debug_ms = 0/0
debug_mon = 0/0
debug_monc = 0/0
debug_paxos = 0/0
debug_tp = 0/0
debug_auth = 0/0
debug_crypto = 0/0
debug_finisher = 0/0
debug_heartbeatmap = 0/0
debug_perfcounter = 0/0
debug_rgw = 0/0
debug_civetweb = 0/0
debug_javaclient = 0/0
debug_asok = 0/0
debug_throttle = 0/0
debug_refs = 0/0
debug_xio = 0/0
debug_compressor = 0/0
debug_bluestore = 0/0
debug_bluefs = 0/0
debug_bdev = 0/0
debug_kstore = 0/0
debug_rocksdb = 0/0
debug_leveldb = 0/0
debug_memdb = 0/0
debug_kinetic = 0/0
debug_fuse = 0/0
debug_mgr = 0/0
debug_mgrc = 0/0
debug_dpdk = 0/0
debug_eventtrace = 0/0

osd_pool_default_size = 1

#ms_crc_data = false
#ms_crc_header = false
#ms_async_op_threads = 5
osd_op_num_threads_per_shard = 3
osd_op_num_shards = 6
osd_enable_op_tracker = false
filestore_wbthrottle_enable = false
filestore_queue_max_ops = 10240
filestore_queue_max_bytes = 1024000000
filestore_op_threads = 6
filestore_fd_cache_size = 1024
filestore_fd_cache_shards = 32
filestore_ondisk_finisher_threads = 4
filestore_apply_finisher_threads = 4
rbd cache = false
rbd_enable_alloc_hint = false
rbd_op_threads = 4
rbd_cache_writethrough_until_flush = false

建立POOL

分配700G,3副本的空間
計算最佳pg數量的公式
= osd數*100/3

# 400個pg
ceph osd pool create poolname 400 400
# 3副本
ceph osd pool set poolname size 3
# 700G空間，資料共享模式
rbd create poolname/poolname-data --image-shared -s 700000 --stripe-count 24 --stripe-unit 64K
# 掛載
rbd-nbd map poolname/poolname-data
lsb lk
# NAME   MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
# nbd0    43:0    0 683.6G  0 disk