StatefulSet 使用NFS配置動態持久化儲存 - 運維筆記
阿新 • • 發佈：2020-11-13
總體流程：
- 建立 NFS 伺服器。
- 建立 Service Account。用來管控 NFS provisioner 在k8s叢集中執行的許可權。
- 建立 StorageClass。負責建立 PVC 並呼叫 NFS provisioner 進行預定的工作，並關聯 PV 和 PVC。
- 建立 NFS provisioner。有兩個功能,一個是在NFS共享目錄下建立掛載點(volume)；二是建立 PV 並將 PV 與 NFS 掛載點建立關聯。
一、NFS部署（省略）
NFS伺服器為：172.16.60.194
NFS掛載目錄為：/data/storage/mobile-decision-server_data
NFS的配置：
[root@k8s-storage01 ~]# cat /etc/exports /data/storage 172.16.60.197(rw,sync,no_root_squash) /data/storage 172.16.60.193(rw,sync,no_root_squash) /data/storage 172.16.60.48(rw,sync,no_root_squash) /data/storage 172.16.60.120(rw,sync,no_root_squash)
二、StatefulSet使用NFS配置動態持久化儲存
下面是使用NFS作為StatefulSet持久化儲存的操作記錄，分別需要建立nfs-provisioner的rbac、storageclass、nfs-client-provisioner和statefulset的pod。
1）建立nfs-rbac.yaml
[root@k8s-master01 mobile-decision-server]# cat nfs-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-provisioner
namespace: kevin
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-provisioner-runner
namespace: kevin
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["get","create","list", "watch","update"]
- apiGroups: ["extensions"]
resources: ["podsecuritypolicies"]
resourceNames: ["nfs-provisioner"]
verbs: ["use"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-provisioner
subjects:
- kind: ServiceAccount
name: nfs-provisioner
namespace: kevin
roleRef:
kind: ClusterRole
name: nfs-provisioner-runner
apiGroup: rbac.authorization.k8s.io
建立並檢視
[root@k8s-master01 mobile-decision-server]# kubectl apply -f nfs-rbac.yaml [root@k8s-master01 mobile-decision-server]# kubectl get sa -n kevin|grep nfs nfs-provisioner 1 7h7m [root@k8s-master01 mobile-decision-server]# kubectl get clusterrole -n kevin|grep nfs nfs-provisioner-runner 7h7m [root@k8s-master01 mobile-decision-server]# kubectl get clusterrolebinding -n kevin|grep nfs run-nfs-provisioner 7h7m
2）建立nfs-class.yaml
[root@k8s-master01 mobile-decision-server]# cat nfs-class.yaml apiVersion: storage.k8s.io/v1beta1 kind: StorageClass metadata: name: managed-nfs-storage namespace: kevin provisioner: mobile-decision-server/nfs reclaimPolicy: Retain
建立
[root@k8s-master01 mobile-decision-server]# kubectl apply -f nfs-class.yaml
3）建立mobile-decision-server-nfs.yml
PROVISIONER_NAME的值一定要和StorageClass中的provisioner相等！
/persistentvolumes的mountPath 掛載點不能修改！
[root@k8s-master01 mobile-decision-server]# cat mobile-decision-server-nfs.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-client-provisioner
namespace: kevin
spec:
replicas: 1
selector:
matchLabels:
app: nfs-client-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccount: nfs-provisioner
containers:
- name: nfs-client-provisioner
image: registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner
imagePullPolicy: IfNotPresent
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: mobile-decision-server/nfs
- name: NFS_SERVER
value: 172.16.60.194
- name: NFS_PATH
value: /data/storage/mobile-decision-server_data
volumes:
- name: nfs-client-root
nfs:
server: 172.16.60.194
path: /data/storage/mobile-decision-server_data
建立並檢視
[root@k8s-master01 mobile-decision-server]# kubectl apply -f mobile-decision-server-nfs.yml [root@k8s-master01 mobile-decision-server]# kubectl get pods -n kevin|grep nfs nfs-client-provisioner-6d78cd7874-krpvj 1/1 Running 0 5h3m
4）建立statefulset
[root@k8s-master01 mobile-decision-server]# cat mobile-decision-server.yml
apiVersion: v1
kind: Service
metadata:
name: mobile-decision-server
namespace: kevin
labels:
app: mobile-decision-server
spec:
clusterIP: None
selector:
app: mobile-decision-server
ports:
- port: 8080
name: server
targetPort: 8080
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mobile-decision-server
namespace: kevin
spec:
serviceName: mobile-decision-server
replicas: 2
selector:
matchLabels:
app: mobile-decision-server
template:
metadata:
labels:
app: mobile-decision-server
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: nodetype
operator: In
values:
- mobile-decision-server
serviceAccount: nfs-provisioner
containers:
- name: mobile-decision-server
image: 172.16.60.196/finhub/mobile-decision-server__20
imagePullPolicy: Always
ports:
- name: dserverport
containerPort: 8080
resources:
requests:
cpu: 200m
memory: 600Mi
limits:
cpu: 200m
memory: 1024Mi
lifecycle:
postStart:
exec:
command: ["/bin/sh","-c","touch /tmp/health"]
preStop:
exec:
command: ["/bin/sh","-c","kill 1"]
livenessProbe:
exec:
command: ["test","-e","/tmp/health"]
initialDelaySeconds: 5
timeoutSeconds: 5
periodSeconds: 10
readinessProbe:
tcpSocket:
port: dserverport
initialDelaySeconds: 15
timeoutSeconds: 5
periodSeconds: 20
volumeMounts:
- name: data
mountPath: /onestop/app
volumeClaimTemplates:
- metadata:
name: data
annotations:
volume.beta.kubernetes.io/storage-class: managed-nfs-storage
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 20Gi
建立並檢視
[root@k8s-master01 mobile-decision-server]# kubectl apply -f mobile-decision-server.yml [root@k8s-master01 mobile-decision-server]# kubectl get pods -n kevin|grep mobile-decision-server mobile-decision-server-0 1/1 Running 0 5h4m mobile-decision-server-1 1/1 Running 0 5h4m
檢視pv和pvc
[root@k8s-master01 mobile-decision-server]# kubectl get pvc -n kevin NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE data-mobile-decision-server-0 Bound pvc-9433d725-bc0a-4fae-a2dc-5c22b2c3928a 20Gi RWX managed-nfs-storage 5h10m data-mobile-decision-server-1 Bound pvc-3615cf99-31ea-491b-8808-8800057d0a21 20Gi RWX managed-nfs-storage 5h9m [root@k8s-master01 mobile-decision-server]# kubectl get pv NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE pvc-3615cf99-31ea-491b-8808-8800057d0a21 20Gi RWX Delete Bound kevin/data-mobile-decision-server-1 managed-nfs-storage 5h9m pvc-9433d725-bc0a-4fae-a2dc-5c22b2c3928a 20Gi RWX Delete
5）檢視NFS共享目錄
只要pvc不刪除，NFS共享目錄下的資料夾名稱不會改變。
[root@k8s-storage01 ~]# cd /data/storage/mobile-decision-server_data/ [root@k8s-storage01 mobile-decision-server_data]# ll total 8 drwxrwxrwx 4 root root 4096 Oct 10 11:53 kevin-data-mobile-decision-server-0-pvc-9433d725-bc0a-4fae-a2dc-5c22b2c3928a drwxrwxrwx 4 root root 4096 Oct 10 11:53 kevin-data-mobile-decision-server-1-pvc-3615cf99-31ea-491b-8808-8800057d0a21