K8S系列第八篇（Service、EndPoints以及高可用kubeadm部署）
阿新 • • 發佈：2021-08-08
更多精彩內容請關注微信公眾號：新猿技術生態圈
更多精彩內容請關注微信公眾號：新猿技術生態圈
更多精彩內容請關注微信公眾號：新猿技術生態圈
Endpoints
名稱空間級資源，如果endpoints和service是同一個名字，那麼就自動關聯。
功能一：與service做負載均衡
[root@k8s ~]# kubectl describe svc Name: kubernetes Namespace: default Labels: component=apiserver provider=kubernetes Annotations: <none> Selector: <none> Type: ClusterIP IP Family Policy: SingleStack IP Families: IPv4 IP: 10.96.0.1 IPs: 10.96.0.1 Port: https 443/TCP TargetPort: 6443/TCP Endpoints: 192.168.15.201:6443 Session Affinity: None Events: <none>
功能二：將外部服務引入叢集
案例
# 先在本機建立一個外部的服務mysql
[root@k8s endpoints]# docker run -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 mysql:5.7
c34bab6ad37f46bae59ef2ee712e8430c53142d30a53119e9912407fd540ad61
# 埠3306，密碼如上
kind: Endpoints apiVersion: v1 metadata: namespace: default name: test-endpoints subsets: - addresses: # 代理ip - ip: 192.168.15.201 ports: - port: 3306 # 服務的埠 protocol: TCP name: http --- kind: Service apiVersion: v1 metadata: name: test-endpoints # 這裡的名稱要和上面一樣才能關聯 namespace: default spec: ports: - port: 3306 targetPort: 3306 protocol: TCP name: http --- kind: Deployment # 提供一個mysql的客戶端 apiVersion: apps/v1 metadata: name: mysql namespace: default spec: selector: matchLabels: app: mysql-v1 template: metadata: labels: app: mysql-v1 spec: containers: - name: mysql image: mysql:5.7 env: - name: MYSQL_ROOT_PASSWORD value: "123456"
# 部署endpoints檔案 [root@k8s endpoints]# kubectl apply -f endpoints.yaml endpoints/test-endpoints created service/test-endpoints created deployment.apps/mysql created [root@k8s endpoints]# kubectl get -f endpoints.yaml NAME ENDPOINTS AGE endpoints/test-endpoints 192.168.15.201:3306 8s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/test-endpoints ClusterIP 10.106.61.144 <none> 3306/TCP 8s NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/mysql 1/1 1 1 8s
更多精彩內容請關注微信公眾號：新猿技術生態圈
更多精彩內容請關注微信公眾號：新猿技術生態圈
更多精彩內容請關注微信公眾號：新猿技術生態圈
# 進入部署的專案
[root@k8s endpoints]# kubectl exec -it mysql-578666457d-g8856 -- bash
# 連結這個叢集內部的ip
root@mysql-578666457d-g8856:/# mysql -uroot -p123456 -h10.106.61.144
mysql> create database db01;
Query OK, 1 row affected (0.01 sec)
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| db01 |
| mysql |
| performance_schema |
| sys |
+--------------------+
5 rows in set (0.00 sec)
# 進入外面的docker的mysql
[root@k8s endpoints]# docker exec -it c34bab6ad37f bash
root@c34bab6ad37f:/# mysql -uroot -p123456
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| db01 |
| mysql |
| performance_schema |
| sys |
+--------------------+
5 rows in set (0.00 sec)
# 發現剛才建立的db01在這裡，說明Endpoints成功的代理了mysql服務
服務健康檢查
配置清單
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: test-deployment
spec:
selector:
matchLabels:
app: nginx-v1
template:
metadata:
labels:
app: nginx-v1
spec:
containers:
- name: nginx
image: nginx
lifecycle: # 回撥HOOK
postStart: # 建立Pod前啟動
exec: # 第一種方式，使用較多
command:
- "/bin/sh"
- "-c"
- "touch /root/1.txt"
httpGet: # 第二種方式（使用少）
port: 80
path: / # httpGet的請求必須返回是200才認為是成功的
tcpSocket: # 第三種方式（使用少）
port: 80
preStop: # 刪除Pod前啟動
exec:
command:
- "/bin/sh"
- "-c"
- "echo 123 > /root/1.txt"
livenessProbe:
exec:
command:
- "bin/bash"
- "-c"
- "cat /usr/share/nginx/html/index.php"
initialDelaySeconds: 0 # 執行延遲時間
periodSeconds: 3 # 探測頻率
timeoutSeconds: 1 # 超時時間
successThreshold: 1 # 探測成功多少次為成功
failureThreshold: 3 # 探測失敗多少次為失敗
readinessProbe:
tcpSocket:
port: 80
initialDelaySeconds: 30 # 專案比較大的時候給大一點
periodSeconds: 1 # 就緒性的敏感度設定較大，使用者體驗較好
timeoutSeconds: 1
successThreshold: 3
failureThreshold: 1
回撥HOOK--lifecycle
啟動時的執行函式為postStart，執行的方式有三種，分別是exec、httpGet、tcpSocket，但是httpGet需要請求到200才會返回成功，否則失敗。
結束時的執行函式為preStop，執行方式與上面類似。
存活性--livenessProbe
存活性這裡一般用exec的形式來檢查，生產環境一般設定如下
livenessProbe:
exec:
command:
- "bin/bash"
- "-c"
- "cat /usr/share/nginx/html/index.php"
initialDelaySeconds: 0 # 執行延遲時間，一般立即執行
periodSeconds: 3 # 探測頻率，三秒探測一次
timeoutSeconds: 1 # 超時時間
successThreshold: 1 # 探測成功多少次為成功
failureThreshold: 3 # 探測失敗多少次為失敗
就緒性--readinessProbe
就緒性這裡一般是通過檢查埠的形式來配置
readinessProbe:
tcpSocket:
port: 80
initialDelaySeconds: 30 # 專案比較大的時候給大一點
periodSeconds: 1 # 就緒性的敏感度設定較大，使用者體驗較好
timeoutSeconds: 1 # 超時時間
successThreshold: 3 # 三次成功為成功
failureThreshold: 1 # 一次失敗剔除
綜合案例--wordpress部落格專案
# 資料庫服務部署
# 資料庫名稱空間建立
apiVersion: v1
kind: Namespace
metadata:
name: mysql
---
# 資料庫控制器建立
apiVersion: apps/v1
kind: Deployment
metadata:
name: mysql
namespace: mysql
spec:
selector:
matchLabels:
app: mysql
template:
metadata:
labels:
app: mysql
spec:
containers:
- name: mysql
image: mysql:5.7
env:
- name: MYSQL_ROOT_PASSWORD
value: "123456"
- name: MYSQL_DATABASE
value: wordpress
livenessProbe: # 存活性檢查
exec:
command:
- "/bin/bash"
- "-c"
- "cat /etc/mysql/my.cnf"
initialDelaySeconds: 0
periodSeconds: 3
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe: # 就緒性檢查
tcpSocket:
port: 3306
initialDelaySeconds: 20
periodSeconds: 1
successThreshold: 3
failureThreshold: 1
timeoutSeconds: 1
---
# 給資料庫配置Service
apiVersion: v1
kind: Service
metadata:
name: mysql
namespace: mysql
spec:
selector:
app: mysql
ports:
- port: 3306
targetPort: 3306
type: NodePort
# 資料庫部署完畢
---
# 建立專案的名稱空間
apiVersion: v1
kind: Namespace
metadata:
namespace: wordpress
name: wordpress
---
# 建立專案的控制器
apiVersion: apps/v1
kind: Deployment
metadata:
name: wordpress
namespace: wordpress
spec:
selector:
matchLabels:
app: wordpress
template:
metadata:
labels:
app: wordpress
spec:
containers:
- name: php
image: alvinos/php:wordpress-v2
imagePullPolicy: Always
livenessProbe:
exec:
command:
- "/bin/bash"
- "-c"
- "ps -ef | grep php"
initialDelaySeconds: 0
periodSeconds: 3
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 1
readinessProbe:
tcpSocket:
port: 9000
initialDelaySeconds: 20
periodSeconds: 1
timeoutSeconds: 1
successThreshold: 3
failureThreshold: 1
- name: nginx
image: alvinos/nginx:wordpress-v2
imagePullPolicy: Always
livenessProbe:
exec:
command:
- "/bin/bash"
- "-c"
- "cat /etc/nginx/nginx.conf"
initialDelaySeconds: 0
periodSeconds: 3
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 1
readinessProbe:
tcpSocket:
port: 80
initialDelaySeconds: 10
periodSeconds: 1
timeoutSeconds: 1
successThreshold: 3
failureThreshold: 1
# 控制器部署完畢
---
# 部署控制器Service
apiVersion: v1
kind: Service
metadata:
name: wordpress
namespace: wordpress
spec:
selector:
app: wordpress
ports:
- port: 80
targetPort: 80
name: http
nodePort: 30080
- port: 443
targetPort: 443
name: https
type: NodePort
cluster.local想要修改可以修改這裡
[root@k8s wordpress]# grep -ro "cluster.local" /etc/kubernetes/
/etc/kubernetes/manifests/kube-apiserver.yaml:cluster.local
ADM的api高可用
匯出初始化檔案，做修改
[root@localhost ~]# kubeadm config print init-defaults > init-config.yaml
[root@localhost ~]# cat init-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 1.2.3.4
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: node
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: 1.21.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}
修改
INIT_IP=`hostname -i`
INIT_HOST=`hostname`
cat > init-config.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: ${INIT_IP} # 當前的主機ip
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: ${INIT_HOST} # 對應的主機名
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
certSANs:
- 192.168.15.59 # 高可用的虛擬IP
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.15.59:8443
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-shanghai.aliyuncs.com/baim0os # 自己的映象倉庫
kind: ClusterConfiguration
kubernetesVersion: 1.21.3
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16
serviceSubnet: 10.96.0.0/12
scheduler: {}
EOF
安裝高可用軟體
# 三臺master節點都需要安裝
# keeplived + haproxy
[root@k8s-m-01 ~]# yum install -y keepalived haproxy
# 修改keepalived配置檔案
# 根據節點的不同，修改的配置也不同
mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak
cd /etc/keepalived
KUBE_APISERVER_IP=`hostname -i`
cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_kubernetes {
script "/etc/keepalived/check_kubernetes.sh"
interval 2
weight -5
fall 3
rise 2
}
vrrp_instance VI_1 {
state MASTER
interface eth0
mcast_src_ip ${KUBE_APISERVER_IP}
virtual_router_id 51
priority 100
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.15.59
}
}
EOF
[root@k8s-m-01 /etc/keepalived]# systemctl enable --now keepalived
# 修改haproxy配置檔案
# 高可用軟體
cat > /etc/haproxy/haproxy.cfg <<EOF
global
maxconn 2000
ulimit-n 16384
log 127.0.0.1 local0 err
stats timeout 30s
defaults
log global
mode http
option httplog
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-request 15s
timeout http-keep-alive 15s
frontend monitor-in
bind *:33305
mode http
option httplog
monitor-uri /monitor
listen stats
bind *:8006
mode http
stats enable
stats hide-version
stats uri /stats
stats refresh 30s
stats realm Haproxy\ Statistics
stats auth admin:admin
frontend k8s-master
bind 0.0.0.0:8443
bind 127.0.0.1:8443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend k8s-master
backend k8s-master
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server m01 192.168.15.51:6443 check inter 2000 fall 2 rise 2 weight 100
server m02 192.168.15.52:6443 check inter 2000 fall 2 rise 2 weight 100
server m03 192.168.15.53:6443 check inter 2000 fall 2 rise 2 weight 100
EOF
[root@k8s-m-01 /etc/keepalived]# systemctl enable --now haproxy.service
Created symlink from /etc/systemd/system/multi-user.target.wants/haproxy.service to /usr/lib/systemd/system/haproxy.service.
初始化叢集
kubeadm init --config init-config.yaml --upload-certs
# 主節點命令複製下來
kubeadm join 192.168.15.59:8443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:b22691a3783c7f1a3544006e64907418476b6942393dffa02b3b0f20cb46a083 \
--control-plane --certificate-key 2e222d296099e3c4656dd9aa12d81b5bbbd0a3f2f13d6d3a9252334034785af1
# 從節點命令複製下來
kubeadm join 192.168.15.59:8443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:b22691a3783c7f1a3544006e64907418476b6942393dffa02b3b0f20cb46a083
# 啟動
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
安裝網路外掛calico
# 下載calico
curl https://docs.projectcalico.org/manifests/calico.yaml -O
# 部署calico
kubectl apply -f calico.yaml
各節點執行加入命令
# 設定叢集角色
kubectl label nodes n01 node-role.kubernetes.io/node=n01
kubectl label nodes n02 node-role.kubernetes.io/node=n02
# 檢視叢集狀態
[root@m01 ~]# kubectl get nodes
[root@m01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
m01 Ready control-plane,master 36m v1.21.3
m02 Ready control-plane,master 6m47s v1.21.3
m03 Ready control-plane,master 5m50s v1.21.3
n01 Ready node 5m v1.21.3
n02 Ready node 4m42s v1.21.3
更多精彩內容請關注微信公眾號：新猿技術生態圈
更多精彩內容請關注微信公眾號：新猿技術生態圈
更多精彩內容請關注微信公眾號：新猿技術生態圈