1. 建立儲存池

ceph-cluster]$ ceph osd pool create kube-cluster 64
ceph-cluster]$ ceph osd pool application enable kube-cluster rbd

2. 授權 ceph 使用者

ceph-cluster]$ ceph auth get-or-create client.kube mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=kube-cluster' -o ceph.client.kube.keyring 
 

3. 獲取 ceph 管理員的資訊並編碼，提供給 k8s 叢集管理 ceph 叢集

ceph-cluster]$ ceph auth get-key client.admin|base64
QVFDb21CWmYzQnZ0Q3hBQWxVOFlWanRkdTRMaitJblBlOHRYcUE9PQ==

4. 獲取 ceph 普通使用者的資訊編碼，提供給 pod 使用

ceph-cluster]$ ceph auth get-key client.kube|base64
QVFEczFCZGZhbzM2TkJBQURzZFlyUjRxbHhYTmF3dEoyUlBVT2c9PQ==

5. 建立 Secret

ceph]# cat secret-cluster.yaml
---
apiVersion: v1
kind: Secret
metadata:
  name: ceph-admin-secret
  namespace: kube-system
data:
  key: QVFDb21CWmYzQnZ0Q3hBQWxVOFlWanRkdTRMaitJblBlOHRYcUE9PQ==
type: "kubernetes.io/rbd"
---
apiVersion: v1
kind: Secret
metadata:
  name: ceph-kube-secret
  namespace: default
data:
  key: QVFEczFCZGZhbzM2TkJBQURzZFlyUjRxbHhYTmF3dEoyUlBVT2c9PQ==
type: "kubernetes.io/rbd"
 

6. 建立 StorageClass 儲存類物件

ceph]# cat ceph-storageclass.yaml 
---
apiVersion: storage.k8s.io/v1beta1
kind: StorageClass
metadata:
  name: rbd-dynamic
  annotations:
    storageclass.beta.kubernetes.io/is-defautl-class: "true"
provisioner: kubernetes.io/rbd
parameters:
  monitors: 192.168.124.161:6789,192.168.124.162:6789,192.168.124.163:6789
  adminId: admin
  adminSecretName: ceph-admin-secret
  adminSecretNamespace: kube-system
  pool: kube-cluster
  userId: kube
  userSecretName: ceph-kube-secret

注意： 動態供給要求 kube-conntroller-manager 所在的節點上擁有 rbd 命令（安裝 ceph-common 程式即可）, 而以 kubeadm 部署的 以Pod 形式執行的 kube-controller-manager 在其容器內部不具有此程式，且無法額外安裝，因此需要以外部 external-provision 的方式供給rbd管理工具

7. 建立測試 Pod

ceph]# cat rbd-pod-test.yaml
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rbd-claim
spec:
  storageClassName: rbd-dynamic
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 2Gi
---
kind: Pod
apiVersion: v1
metadata:
  name: rbd-pod
spec:
  containers:
  - name: busybox
    image: busybox
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "touch /mnt/SUCCESS && sleep 3600"
    volumeMounts:
      - name: rbd-pvc
        mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
    - name: rbd-pvc
      persistentVolumeClaim:
        claimName: rbd-claim

8. 驗證

ceph]# kubectl  get pod -o wide
NAME      READY   STATUS    RESTARTS   AGE   IP           NODE              NOMINATED NODE   READINESS GATES
rbd-pod   1/1     Running   0          41s   172.20.2.8   192.168.124.221   <none>           <none>

~]# rbd showmapped
id pool         image                                                       snap device    
0  kube-cluster kubernetes-dynamic-pvc-1017c477-162c-491f-9027-56bcfcd75919 -    /dev/rbd0 

ceph]# kubectl  exec -it pod/rbd-pod -- ls /mnt
SUCCESS     lost+found