k8s實現elk日誌收集系統部署
阿新 • • 發佈：2020-10-14
目錄
k8s實現elk日誌收集系統
說明：es搭建叢集，3臺master，3臺node，node角色分別為熱資料，溫資料，冷資料
kibana進行資料展示，同時連線es的工具使用es-head。
此處使用deployment+service結合搭建,logstash還未進行試驗，等我繼續更新。。。
master節點搭建
######### 建立名稱空間 ########### kind: Namespace apiVersion: v1 metadata: name: elastic --- ######## service型別為nodeport用以暴露es埠供外部訪問 ######## kind: Service apiVersion: v1 metadata: name: es-cluster-nodeport namespace: elastic spec: type: NodePort selector: app: es-cluster ports: - port: 9200 targetPort: 9200 nodePort: 32700 --- ####### es叢集間通訊埠，使用service進行暴露，但不像nodeport可對叢集外伺服器提供訪問 ############# kind: Service apiVersion: v1 metadata: name: es-svc namespace: elastic labels: app: es-cluster annotations: es.kubernetes.io/es: es-cluster spec: selector: app: es-cluster role: master ports: - port: 9300 targetPort: 9300 --- ###### es的master建立 ############################# kind: Deployment # 資源型別 apiVersion: apps/v1 # 版本 metadata: # 元資料 namespace: elastic # 名稱空間 name: es-cluster-master # deployment型別的元資料名 labels: # 標籤 app: es-cluster-master # 定義deployment的標籤 role: master spec: # 說明 replicas: 3 # 副本數為3,表示建立3個pod selector: # 選擇器 matchLabels: # 精確選擇 app: es-cluster # 選擇標籤為 app: es-cluster role: master # 選擇標籤為 role: master template: # 模板 metadata: # 元資料資訊 labels: # 標籤 app: es-cluster # 此模板的資訊 role: master spec: # 模板對應的說明 containers: # 容器 - name: elasticsearch-master # 名稱 image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1 # 映象 ports: # 埠 - containerPort: 9200 # 這裡的埠不提供暴露，僅說明 protocol: TCP - containerPort: 9300 protocol: TCP lifecycle: # 生命週期，也是回撥鉤子 postStart: # 當容器啟動後執行的操作 exec: # 執行命令 command: - "/bin/sh" - "-c" - "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;" securityContext: # 安全上下文，指具備對系統的操作許可權 privileged: true readinessProbe: # 就緒性探測 httpGet: # 探測9200埠是否就緒 port: 9200 volumeMounts: # 儲存性掛載 - mountPath: /usr/share/elasticsearch/data # 容器中需要掛載的路徑 name: es-data # 掛載到的地方，這個名稱對應下方的掛載 env: # 此處定義的環境變數相當於在命令列執行命令，替代了配置檔案的操作 - name: "cluster.name" value: "es-cluster" - name: "bootstrap.memory_lock" value: "true" - name: "discovery.zen.ping.unicast.hosts" value: "es-svc" - name: "discovery.zen.minimum_master_nodes" value: "2" - name: "discovery.zen.ping_timeout" value: "5s" - name: "node.master" value: "true" - name: "node.data" value: "false" - name: "ES_JAVA_OPTS" value: "-Xms256m -Xmx256m" - name: "http.cors.enabled" value: "true" - name: "http.cors.allow-origin" value: "*" volumes: # 對應上方的掛載名 - name: es-data emptyDir: {}
head外掛
kind: Deployment apiVersion: apps/v1 metadata: namespace: elastic name: es-head spec: selector: matchLabels: # deployment精確匹配選擇對應的模板 app: es-head template: metadata: labels: app: es-head # 模板標籤資訊，供deployment選擇 spec: containers: - name: head image: alvinos/elasticsearch-head ports: - containerPort: 9100 protocol: TCP --- ##### 使用nodeport型的service，將pod中的埠進行暴露 ########## kind: Service apiVersion: v1 metadata: name: es-head-svc namespace: elastic spec: type: NodePort selector: app: es-head ports: - port: 9100 targetPort: 9100 nodePort: 32701
node節點--hot
kind: Deployment apiVersion: apps/v1 metadata: name: es-cluster-data-hot namespace: elastic labels: # deployment的標籤 app: es-cluster-data-hot role: data spec: selector: matchLabels: # deployment選擇器精確匹配模板中的資訊 app: es-cluster role: data datatype: hot template: metadata: labels: # 供上方deployment選擇器選擇 app: es-cluster role: data datatype: hot spec: containers: - name: es-cluster-hot image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1 ports: - containerPort: 9200 protocol: TCP - containerPort: 9300 protocol: TCP env: # 環境變數 - name: "cluster.name" value: "es-cluster" - name: "bootstrap.memory_lock" value: "true" - name: "discovery.zen.ping.unicast.hosts" value: "es-svc" - name: "node.master" value: "false" - name: "node.data" value: "true" - name: "ES_JAVA_OPTS" value: "-Xms256m -Xmx256m" - name: "node.attr.box_type" value: "hot" # 指定為hot資料型別 lifecycle: postStart: # 容器啟動後的操作 exec: command: - "/bin/sh" - "-c" - "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;" securityContext: privileged: true volumeMounts: - mountPath: /usr/share/elasticsearch/data name: es-data volumes: - name: es-data emptyDir: {}
node節點--warm
kind: Deployment
apiVersion: apps/v1
metadata:
name: es-cluster-data-warm
namespace: elastic
labels:
app: es-cluster-data-warm
role: data
spec:
selector:
matchLabels:
app: es-cluster
role: data
datatype: warm
template:
metadata:
labels:
app: es-cluster
role: data
datatype: warm
spec:
containers:
- name: es-cluster-warm
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
env:
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "node.master"
value: "false"
- name: "node.data"
value: "true"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "node.attr.box_type"
value: "warm"
lifecycle:
postStart:
exec:
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext:
privileged: true
readinessProbe:
httpGet:
port: 9200
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: es-data
volumes:
- name: es-data
emptyDir: {}
node節點--cold
kind: Deployment
apiVersion: apps/v1
metadata:
name: es-cluster-data-cold
namespace: elastic
labels:
app: es-cluster-data-cold
role: data
spec:
selector:
matchLabels:
app: es-cluster
role: data
datatype: cold
template:
metadata:
labels:
app: es-cluster
role: data
datatype: cold
spec:
containers:
- name: es-cluster-cold
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
env:
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "node.master"
value: "false"
- name: "node.data"
value: "true"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "node.attr.box_type"
value: "cold"
lifecycle:
postStart:
exec:
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext:
privileged: true
readinessProbe:
httpGet:
port: 9200
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: es-data
volumes:
- name: es-data
emptyDir: {}