LVS實現Kubernetes叢集高可用
阿新 • • 發佈：2019-02-01
一、環境說明
1.作業系統：centos 7.5 x86_64
2.三臺k8s-master 3.兩臺lvs
k8s01：10.10.10.206 lb01：10.10.10.219
k8s02：10.10.10.207 lb02：10.10.10.220
k8s03：10.10.10.208 vipIP：10.10.10.203
二、系統配置
三臺k8s-master
1）建立生成K8S csr的JSON配置檔案，然後生成kubernetes*.pem
# cd /root/ssl # cat > kubernetes-csr.json <<EOF { "CN": "kubernetes", "hosts": [ "127.0.0.1", "10.10.10.206", "10.10.10.207", "10.10.10.208", "10.10.10.203" "10.1.0.1" "10.2.0.1", "localhost", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "BeiJing", "L": "BeiJing", "O": "k8s", "OU": "System" } ] } EOF
注：10.1.0.1地址為service-cluster網段中第一個ip，10.2.0.1地址為cluster-cidr網段中第一個ip
2）在“Kubernetes+Docker+Calico叢集安裝配置”文件中master和各node節點所有配置連線時使用https://192.168.168.2:6443部分都替換為https://10.10.10.203:6443
3）LVS部分配置
a.配置Linux核心引數
# vi /etc/sysctl.conf net.ipv4.ip_forward = 1 net.ipv4.conf.lo.arp_ignore = 1 net.ipv4.conf.lo.arp_announce = 2 net.ipv4.conf.all.arp_ignore = 1 net.ipv4.conf.all.arp_announce = 2 net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_local_port_range = 30000 60999 net.netfilter.nf_conntrack_max = 26214400 net.netfilter.nf_conntrack_tcp_timeout_established = 86400 net.netfilter.nf_conntrack_tcp_timeout_close_wait = 3600 # sysctl -p
b.配置VIP地址繫結在lo網絡卡上
# mkdir /opt/scripts/lvs_real.sh
# vi /opt/scripts/lvs_real.sh
#!/bin/bash
#description: Config realserver
VIP=10.10.10.203
. /etc/rc.d/init.d/functions
case "$1" in
start)
ifconfig lo:0 $VIP netmask 255.255.255.255 broadcast $VIP
/sbin/route add -host $VIP dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null 2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0 down
route del $VIP >/dev/null 2>&1
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
echo "RealServer Stoped"
;;
status)
#Status of LVS-DR real server.
islothere=`/sbin/ifconfig lo:0 | grep $VIP`
isrothere=`netstat -rn | grep "lo:0" | grep $VIP`
if [ ! "$islothere" -o ! "isrothere" ];then
# Either the route or the lo:0 device
# not found.
echo "LVS-DR real server Stopped."
else
echo "LVS-DR Running."
fi
;;
*)
#Invalid entry.
echo "$0: Usage: $0 {start|status|stop}"
exit 1
;;
esac
exit 0
# chmod +x /opt/scripts/lvs_real.sh
# /opt/scripts/lvs_real.sh start4）檢視lo網口繫結VIP狀態
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet 10.10.10.203/32 brd 10.10.10.203 scope global lo:0
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever5）配置lvs_real.sh指令碼開機自動執行
# vi /etc/rc.d/rc.local
#!/bin/bash
# THIS FILE IS ADDED FOR COMPATIBILITY PURPOSES
#
# It is highly advisable to create own systemd services or udev rules
# to run scripts during boot instead of using this file.
#
# In contrast to previous versions due to parallel execution during boot
# this script will NOT be run after all other services.
#
# Please note that you must run 'chmod +x /etc/rc.d/rc.local' to ensure
# that this script will be executed during boot.
#touch /var/lock/subsys/local
bash /opt/scripts/lvs_real.sh startchmod +x /etc/rc.d/rc.local
6）編輯rc-local.service在末尾新增[Install]部分
# vi /usr/lib/systemd/system/rc-local.service
# This file is part of systemd.
#
# systemd is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
# This unit gets pulled automatically into multi-user.target by
# systemd-rc-local-generator if /etc/rc.d/rc.local is executable.
[Unit]
Description=/etc/rc.d/rc.local Compatibility
ConditionFileIsExecutable=/etc/rc.d/rc.local
After=network.target
[Service]
Type=forking
ExecStart=/etc/rc.d/rc.local start
TimeoutSec=0
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target設定開機啟動
systemctl daemon-reload
systemctl enable rc-local.service
systemctl start rc-local.service2.lvs+keepalived配置，配置前關閉selinux和firewalld
在lb01和lb02主機上yum -y install ipvsadm keepalived
1）lb01 keepalived配置檔案
# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_k8s
}
vrrp_script CheckKeepalived {
script "/etc/keepalived/chk_keepalived.sh"
interval 3
weight -10
fall 2
rise 2
}
vrrp_instance VI_1 {
state MASTER
interface ens32
virtual_router_id 66
priority 100
advert_int 1
vrrp_garp_master_repeat 5
vrrp_garp_master_refresh 10
authentication {
auth_type PASS
auth_pass 6666
}
virtual_ipaddress {
10.10.10.203 dev ens32 label ens32:vip
}
track_script {
CheckKeepalived
}
}
virtual_server 10.10.10.203 6443 {
delay_loop 6
lb_algo rr
lb_kind DR
# persistence_timeout 0
protocol TCP
real_server 10.10.10.206 6443 {
weight 10
TCP_CHECK {
connect_timeout 10
}
}
real_server 10.10.10.207 6443 {
weight 10
TCP_CHECK {
connect_timeout 10
}
}
real_server 10.10.10.208 6443 {
weight 10
TCP_CHECK {
connect_timeout 10
}
}
}注：此例lvs繫結網絡卡號為ens32
2）lb02 keepalived配置檔案
# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_k8s
}
vrrp_script CheckKeepalived {
script "/etc/keepalived/chk_keepalived.sh"
interval 3
weight -10
fall 2
rise 2
}
vrrp_instance VI_1 {
state BACKUP
interface ens32
virtual_router_id 66
priority 95
advert_int 1
authentication {
auth_type PASS
auth_pass 6666
}
virtual_ipaddress {
10.10.10.203 dev ens32 label ens32:vip
}
track_script {
CheckKeepalived
}
}
virtual_server 10.10.10.203 6443 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 0
protocol TCP
real_server 10.10.10.206 6443 {
weight 10
TCP_CHECK {
connect_timeout 10
}
}
real_server 10.10.10.207 6443 {
weight 10
TCP_CHECK {
connect_timeout 10
}
}
real_server 10.10.10.208 6443 {
weight 10
TCP_CHECK {
connect_timeout 10
}
}
}3）keepalived故障檢測檔案
# vi /etc/keepalived/chk_keepalived.sh
#!/bin/bash
keepalived_counter=$(ps -C keepalived --no-heading|wc -l)
if [ "${keepalived_counter}" = "0" ]; then
/usr/sbin/keepalived
fi
# chmod +x /etc/keepalived/chk_keepalived.sh4）Linux 核心引數配置
# vi /etc/sysctl.conf
net.ipv4.ip_forward = 1
net.ipv4.ip_nonlocal_bind = 1
# sysctl -p5）啟動keepalived後檢視網絡卡繫結和ipvsadm狀態
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:50:56:b7:3f:e5 brd ff:ff:ff:ff:ff:ff
inet 10.10.10.219/24 brd 10.10.10.255 scope global noprefixroute ens32
valid_lft forever preferred_lft forever
inet 10.10.10.203/32 scope global ens32:vip
valid_lft forever preferred_lft forever
inet6 fe80::250:56ff:feb7:3fe5/64 scope link
valid_lft forever preferred_lft forever# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.10.10.203:6443 rr
-> 10.10.10.206:6443 Route 10 0 0
-> 10.10.10.207:6443 Route 10 0 0
-> 10.10.10.208:6443 Route 10 0 0