背景使用 EFK 收集 K8S 日誌(K8S 容器執行時是 containerd)

安裝 es 叢集

# 建立目錄
mkdir -p /data/yaml/kube-logging/{elasticsearch,filebeat,kibana}

cd /data/yaml/kube-logging/

# 建立名稱空間
cat > kube-logging.yaml << EOF
kind: Namespace
apiVersion: v1
metadata:
  name: kube-logging
EOF

kubectl apply -f kube-logging.yaml

cd /data/yaml/kube-logging/elasticsearch

# 建立es後端儲存，這裡使用 ceph
cat > es-rbd-sc.yaml << EOF
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: es-rbd-sc
   namespace: kube-logging
provisioner: rbd.csi.ceph.com
parameters:
   clusterID: 24c6d785-eec0-44f7-8ae1-738d05a1c89d
   pool: kubernetes
   csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
   csi.storage.k8s.io/provisioner-secret-namespace: default
   csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
   csi.storage.k8s.io/node-stage-secret-namespace: default
   imageFormat: "2"
   imageFeatures: "layering"
reclaimPolicy: Delete
mountOptions:
   - discard
EOF

kubectl apply -f es-rbd-sc.yaml

# 建立 es svc
cat > elasticsearch_svc.yaml << EOF
kind: Service
apiVersion: v1
metadata:
  name: elasticsearch
  namespace: kube-logging
  labels:
    app: elasticsearch
spec:
  selector:
    app: elasticsearch
  clusterIP: None
  ports:
    - port: 9200
      name: rest
    - port: 9300
      name: inter-node

EOF

kubectl apply -f elasticsearch_svc.yaml

# 建立 es pod
cat > elasticsearch_statefulset.yaml << EOF
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: es-cluster
  namespace: kube-logging
spec:
  serviceName: elasticsearch
  replicas: 3
  selector:
    matchLabels:
      app: elasticsearch
  template:
    metadata:
      labels:
        app: elasticsearch
    spec:
      containers:
      - name: elasticsearch
        image: elasticsearch:7.2.0
        imagePullPolicy: IfNotPresent
        resources:
            limits:
              cpu: 1000m
            requests:
              cpu: 100m
        ports:
        - containerPort: 9200
          name: rest
          protocol: TCP
        - containerPort: 9300
          name: inter-node
          protocol: TCP
        volumeMounts:
        - name: data
          mountPath: /usr/share/elasticsearch/data
        env:
          - name: cluster.name
            value: k8s-logs
          - name: node.name
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: discovery.seed_hosts
            value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"
          - name: cluster.initial_master_nodes
            value: "es-cluster-0,es-cluster-1,es-cluster-2"
          - name: ES_JAVA_OPTS
            value: "-Xms512m -Xmx512m"
      initContainers:
      - name: fix-permissions
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
        securityContext:
          privileged: true
        volumeMounts:
        - name: data
          mountPath: /usr/share/elasticsearch/data
      - name: increase-vm-max-map
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sysctl", "-w", "vm.max_map_count=262144"]
        securityContext:
          privileged: true
      - name: increase-fd-ulimit
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "ulimit -n 65536"]
        securityContext:
          privileged: true
  volumeClaimTemplates:
  - metadata:
      name: data
      labels:
        app: elasticsearch
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: es-rbd-sc
      resources:
        requests:
          storage: 5Gi

kubectl apply -f elasticsearch_statefulset.yaml
 

安裝 Kibana

cd /data/yaml/kube-logging/kibana 

cat > kibana.yaml << EOF
apiVersion: v1
kind: Service
metadata:
  name: kibana
  namespace: kube-logging
  labels:
    app: kibana
spec:
  type: NodePort
  ports:
  - port: 5601
    targetPort: 5601
    nodePort: 30056
  selector:
    app: kibana
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: kube-logging
  labels:
    app: kibana
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kibana
  template:
    metadata:
      labels:
        app: kibana
    spec:
      containers:
      - name: kibana
        image: kibana:7.2.0
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 1000m
          requests:
            cpu: 100m
        env:
          - name: ELASTICSEARCH_URL
            value: http://elasticsearch:9200
        ports:
        - containerPort: 5601

EOF

kubectl apply -f kibana.yaml
 

安裝 filebeat

cd /data/yaml/kube-logging/filebeat

# 配置許可權
cat > rbac.yaml << EOF 
---
# Source: filebeat/templates/filebeat-service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: kube-logging
  labels:
    k8s-app: filebeat

---
# Source: filebeat/templates/filebeat-role.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list

---
# Source: filebeat/templates/filebeat-role-binding.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
- kind: ServiceAccount
  name: filebeat
  namespace: kube-logging
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io

EOF

kubectl apply -f rbac.yaml

# 建立配置
cat > filebeat-cm.yaml << EOF 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-logging
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    setup.ilm.enabled: false
    filebeat.inputs:
    - type: container
      paths:
        - /var/log/containers/*.log
      processors:
        - add_kubernetes_metadata:
            host: ${NODE_NAME}
            matchers:
            - logs_path:
                logs_path: "/var/log/containers/"
      multiline.pattern: '^[[:space:]]+(at|\.{3})\b|^Caused by:'
      multiline.negate: false
      multiline.match: after

    processors:
      - add_cloud_metadata:
      - add_host_metadata:

    cloud.id: ${ELASTIC_CLOUD_ID}
    cloud.auth: ${ELASTIC_CLOUD_AUTH}

    setup.template.name: "k8s"
    setup.template.pattern: "k8s-*"
    setup.template.enabled: false
    # 如果是第一次則不需要, 如果 index-template 已經存在需要更新, 則需要
    setup.template.overwrite: false
    setup.template.settings:
      # 根據收集的日誌量級, 因為日誌會每天一份, 如果一天的日誌量小於 30g, 一個 shard 足夠
      index.number_of_shards: 2
      # 這個日誌並不是那麼重要, 並且如果是單節點的話, 直接設定為 0 個副本
      index.number_of_replicas: 0

    output.elasticsearch:
      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
      # 啟動程序數
      worker: 20
      # 傳送重試的次數取決於max_retries的設定預設為3
      max_retries: 3
      # 單個elasticsearch批量API索引請求的最大事件數。預設是50。
      bulk_max_size: 800
      #index: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
      indices:
        - index: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"

    setup.kibana:
      host: '${KIBANA_HOST:kibana-logging}:${KIBANA_PORT:5601}'

    # 設定 ilm 的 policy life, 日誌保留 45 天, 每 7 天一輪訓, 最大 30g
    setup.ilm:
      policy_file: /etc/indice-lifecycle.json
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-index-rules
  namespace: kube-logging
  labels:
    k8s-app: filebeat
data:
  indice-lifecycle.json: |-
    {
      "policy": {
        "phases": {
          "hot": {
            "actions": {
              "rollover": {
                "max_size": "5GB" ,
                "max_age": "1d"
              }
            }
          },
          "delete": {
            "min_age": "7d",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }

EOF

kubectl apply -f filebeat-cm.yaml

# 建立 pod
cat > filebeat-ds.yaml << EOF
 
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: kube-logging
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: filebeat
        image: elastic/filebeat:7.4.2
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        env:
        - name: ELASTICSEARCH_HOST
          value: "elasticsearch"
        - name: ELASTICSEARCH_PORT
          value: "9200"
        - name: ELASTICSEARCH_USERNAME
          value: 
        - name: ELASTICSEARCH_PASSWORD
          value: 
        - name: ELASTIC_CLOUD_ID
          value:
        - name: ELASTIC_CLOUD_AUTH
          value:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        securityContext:
          runAsUser: 0
        resources:
          requests:
            cpu: 100m
            memory: 500Mi
        volumeMounts:
        - name: config
          mountPath: /etc/filebeat.yml
          readOnly: true
          subPath: filebeat.yml
        - name: rules
          mountPath: /etc/indice-lifecycle.json
          readOnly: true
          subPath: indice-lifecycle.json
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: varlog
          mountPath: /var/log
          readOnly: true
      volumes:
      - name: config
        configMap:
          defaultMode: 0600
          name: filebeat-config
      - name: rules
        configMap:
          defaultMode: 0600
          name: filebeat-index-rules
      - name: varlibdockercontainers
        hostPath:
          path: /var/log/pods
      - name: varlog
        hostPath:
          path: /var/log
      # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
      - name: data
        hostPath:
          path: /var/lib/filebeat-data
          type: DirectoryOrCreate
EOF

kubectl apply -f filebeat-ds.yaml