6.定時清除歷史映象：

萬里長征的最後一步收尾工作，換一種更“原生”的玩法，找一找新感覺，手搓走起.....

參考：https://github.com/kubernetes-sigs/cri-tools/blob/master/docs/crictl.md

（a）與docker情況一致，每個節點都安裝containerd和crictl，使用本地資料卷掛載方式

（b）job一次並行執行和最小完成pod數與工作節點相同數量，pod親和性排程只會考慮生命週期在running狀態的pod

（c）使用pod反親和性與自身相同的標籤不在同一節點，通過控制工作節點數，保證工作節點有且只有一個job，job controller中的控制迴圈負責調諧期望狀態和期望狀態，並不會有死鎖的存在

（d）設定容器重啟策略OnFailure，避免pod漂移，backoffLimit針對容器同樣生效

模板：/script/template/rm-images-job.yaml

apiVersion: batch/v1
kind: CronJob
metadata:
  name: {{name}}
  namespace: devops-tools
spec:
  schedule: "* * */7 * *"
  concurrencyPolicy: Replace
  jobTemplate:
    spec:
      completions: {{workerCount}}
      parallelism: {{workerCount}}
      activeDeadlineSeconds: 100
      backoffLimit: 2
      template:
        metadata:
          labels:
            app: {{name}}
        spec:
          securityContext:
            fsGroup: 0
            runAsUser: 0
          serviceAccount: jenkins-admin
          containers:
            - name: {{name}}
              image: myhub.com/devops-tools/kubectl:1.15.3
              command: [ "/bin/sh", "-c" ]
              args:
                - source /etc/podinfo/labels 2>/dev/null;echo "name:${app}";               
                  docker ps -a | grep ${app} | awk '{print $1}' | xargs docker rm -f $1 | xargs echo "docker rm ";
                  docker image prune -a --filter "label=service=${app}" -f;
                  echo "docker rmi CMD:docker image prune -a --filter \"label=service=${app}\" -f;finished";
                  kubectl config set-credentials jenkins-admin --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token);
                  kubectl config set-cluster kubernetes --insecure-skip-tls-verify=true --server=https://kubernetes.default.svc.cluster.local;
                  kubectl config set-context mycontext --cluster=kubernetes --user=jenkins-admin;
                  kubectl config use-context mycontext;
                  usedimage=$(kubectl get deployment.apps/${app} -n {{branch}} -o go-template --template='{{(index .spec.template.spec.containers 0).image}}');
                  echo "get usedimage CMD:kubectl get deployment.apps/${app} -n master -o go-template --template='{{(index .spec.template.spec.containers 0).image}}'";echo "usedimage:${usedimage}";
                  usedimageid=$(crictl images | grep ${usedimage} | awk -v ORS="," '{print $1":"$2"@"$3}');echo "usedimageid:${usedimageid}";
                  IFS=",";
                  imagesinfo=$(crictl images | grep ${app} | awk -v ORS="," '{print $1":"$2"@"$3}');
                  echo "get imagesinfo CMD:crictl images | grep ${app} | awk -v ORS="," '{print \$1":"\$2"@"\$3}' ";echo "imagesinfo:${imagesinfo}";
                  arr=(${imagesinfo});
                  for(( i=0;i<${#arr[@]};i++)) do
                  IFS="@";
                  tags=(${arr[i]});echo "processing tag:${tags}";
                  if [ "${tags[0]}" != "${usedimages}" -a "${tags[1]}" != "${usedimagesid}" ];then
                  crictl rmi ${arr[i]};echo "crictl rmi:${arr[i]}";
                  fi;
                  done;
              volumeMounts:
                - mountPath: "/var/run/docker.sock"
                  name: "dockersocket"
                  readOnly: false
                - mountPath: "/etc/docker"
                  name: "dockerconfig"
                  readOnly: false
                - mountPath: "/usr/bin/docker"
                  name: "docker"
                  readOnly: false
                - mountPath: "/var/run/containerd/containerd.sock"
                  name: "containerdsocket"
                  readOnly: false
                - mountPath: "/etc/crictl.yaml"
                  name: "crictlconfig"
                  readOnly: false
                - mountPath: "/usr/local/bin/crictl"
                  name: "crictl"
                  readOnly: false
                - name: podinfo
                  mountPath: /etc/podinfo
                  readOnly: false
          restartPolicy: OnFailure
          affinity:
            podAntiAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchExpressions:
                      - key: app
                        operator: In
                        values:
                          - {{name}}
                  topologyKey: kubernetes.io/hostname
          volumes:
            - hostPath:
                path: "/usr/bin/docker"
              name: "docker"
            - hostPath:
                path: "/etc/docker"
              name: "dockerconfig"
            - hostPath:
                path: "/var/run/docker.sock"
              name: "dockersocket"
            - hostPath:
                path: "/usr/local/bin/crictl"
              name: "crictl"
            - hostPath:
                path: "/etc/crictl.yaml"
              name: "crictlconfig"
            - hostPath:
                path: "/var/run/containerd/containerd.sock"
              name: "containerdsocket"
            - name: podinfo
              projected:
                sources:
                  - downwardAPI:
                      items:
                        - path: "labels"
                          fieldRef:
                            fieldPath: metadata.labels
 

常用刪除查詢映象資訊命令：

 # 清除docker容器
$ docker  ps -a | grep ${name} <name:goods-web-test> | awk '{print $1}' |  xargs docker rm -f $1
# 清除docker映象  7天前且未使用的專案映象
$ docker image prune -a --filter "until=$(date +'%Y-%m-%dT%H:%M:%S' --date='-7 days')" --filter "label=service=${name} <label:service=goods-web-test>" -f

# 清除k8s映象 未使用的專案映象
# 1.查詢deployment使用的映象
usedimage=$(kubectl get deployment.apps/${name} <name:goods-web-test> -n master -o go-template --template='{{(index .spec.template.spec.containers 0).image}}')
usedimageid=$(crictl images | grep ${name} <name:goods-web-test> | awk -v ORS="," '{print $1":"$2"@"$3}')
# 2.獲取k8s 所有專案映象
$ IFS=","
$ imagesinfo=$(crictl images | grep ${name} <name:goods-web-test> | awk -v ORS="," '{print $1":"$2"@"$3}')
$ arr=(${imagesinfo})
$ for(( i=0;i<${#arr[@]};i++))
do
  IFS="@"
  tags=(${arr[i]})
  if [ "${tags[0]}" != "${usedimages}" -a "${tags[1]}" != "${usedimagesid}" ];then
    crictl rmi ${arr[i]};
  fi
done;

# 獲取映象標籤
docker image inspect --format='{{json .Config.Labels}}' ae513a47849c

crictl  inspecti  -o go-template --template='{{json .info.imageSpec.config.Labels}}'  779aa7e4e93c4
crictl  inspecti  -o go-template --template='{{json .info.imageSpec.config.Labels.description}}'  779aa7e4e93c4
crictl  inspecti  -o go-template --template='{{index .info.imageSpec.config.Labels "description"}}'  779aa7e4e93c4
 

7.容器與主機時間同步配置：

參考：https://zhuanlan.zhihu.com/p/156757418