高可用keepalived之進階
阿新 • • 發佈:2021-07-03
實現 master/master 的 Keepalived 雙主架構
master/slave的單主架構,同一時間只有一個Keepalived對外提供服務,此主機繁忙,而另一臺主機卻
很空閒,利用率低下,可以使用master/master的雙主架構,解決此問題。
master/master 的雙主架構:
即將兩個或以上VIP分別執行在不同的keepalived伺服器,以實現伺服器並行提供web訪問的目的,提高
伺服器資源利用率
#ha1主機配置 [root@ka1-centos8 ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { [email protected] } notification_email_from keepalived@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id ka1.longxuan.vip vrrp_mcast_group4 224.8.8.188 } vrrp_instance N520 { state MASTER #在另一個主機上為BACKUP interface eth0 virtual_router_id 66 #每個vrrp_instance唯一 priority 100 #在另一個主機上為80 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 #指定vrrp_instance各自的VIP } } vrrp_instance N521 { #新增 VI_2 例項 state BACKUP #在另一個主機上為MASTER interface eth0 virtual_router_id 88 #每個vrrp_instance唯一 priority 80 #在另一個主機上為100 advert_int 1 authentication { auth_type PASS auth_pass 12345678 } virtual_ipaddress { 172.31.0.200 dev eth0 label eth0:2 #指定vrrp_instance各自的VIP } } #ka2主機配置,和ka1配置只需五行不同 [root@ka2-centos8 ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { [email protected] } notification_email_from keepalived@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id ka2.longxuan.vip #修改此行 vrrp_mcast_group4 224.8.8.100 } vrrp_instance N520 { state BACKUP #此修改行為BACKUP interface eth0 virtual_router_id 66 priority 80 #此修改行為80 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 } } vrrp_instance N521 { state MASTER #修改此行為MASTER interface eth0 virtual_router_id 88 priority 100 #修改此行為100 advert_int 1 authentication { auth_type PASS auth_pass 12345678 } virtual_ipaddress { 172.31.0.200 dev eth0 label eth0:2 } }
實戰案例:利用子配置檔案實現master/master的Keepalived雙主架構
範例:ka1配置
[root@centos8 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { acassen } notification_email_from [email protected] smtp_server 192.168.200.1 smtp_connect_timeout 30 router_id LVS_DEVEL } include /etc/keepalived/conf.d/*.conf # 建立子目錄 [root@centos8 ~]# mkdir /etc/keepalived/conf.d/ -p # 專案1-master [root@centos8 ~]# cat /etc/keepalived/conf.d/n520.conf vrrp_instance n520 { state MASTER interface eth0 virtual_router_id 66 priority 100 advert_int 1 #nopreempt #preempt_delay 10 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 } unicast_src_ip 172.31.0.28 unicast_peer{ 172.31.0.48 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } # 專案2-backup [root@centos8 ~]# vim /etc/keepalived/conf.d/n521.conf vrrp_instance n521 { state BACKUP interface eth0 virtual_router_id 88 priority 80 advert_int 1 authentication { auth_type PASS auth_pass 12345678 } virtual_ipaddress { 172.31.0.200 dev eth0 label eth0:2 } unicast_src_ip 172.31.0.28 unicast_peer{ 172.31.0.48 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } # 重啟 [root@centos8 ~]# systemctl restart keepalived 檢查ip [root@centos8 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 00:0c:29:ac:f5:a4 brd ff:ff:ff:ff:ff:ff inet 172.31.0.28/16 brd 172.31.255.255 scope global noprefixroute eth0 valid_lft forever preferred_lft forever inet 172.31.0.188/32 scope global eth0:1 valid_lft forever preferred_lft forever inet 172.31.0.200/32 scope global eth0:2
範例:ka2配置
[root@centos8 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { acassen } notification_email_from [email protected] smtp_server 192.168.200.1 smtp_connect_timeout 30 router_id LVS_DEVEL } include /etc/keepalived/conf.d/*.conf # 建立子目錄 [root@centos8 ~]# mkdir /etc/keepalived/conf.d/ -p # 專案1-backup [root@centos8 ~]# cat /etc/keepalived/conf.d/n520.conf vrrp_instance n520 { state BACKUP interface eth0 virtual_router_id 66 priority 80 advert_int 1 #nopreempt #preempt_delay 10 #如果設定延遲搶佔模式,需要都是BACKUP,不然會造成腦裂 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 } unicast_src_ip 172.31.0.48 unicast_peer{ 172.31.0.28 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } # 專案2-master [root@centos8 ~]# cat //etc/keepalived/conf.d/n521.conf vrrp_instance n521 { state MASTER interface eth0 virtual_router_id 88 priority 80 advert_int 1 authentication { auth_type PASS auth_pass 12345678 } virtual_ipaddress { 172.31.0.200 dev eth0 label eth0:2 } unicast_src_ip 172.31.0.48 unicast_peer{ 172.31.0.28 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } # 重啟keepalived [root@centos8 ~]# systemctl restart keepalived # 檢視ip [root@centos8 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 00:0c:29:16:9a:81 brd ff:ff:ff:ff:ff:ff inet 172.31.0.48/16 brd 172.31.255.255 scope global noprefixroute eth0 valid_lft forever preferred_lft forever inet 172.31.0.200/32 scope global eth0:2 valid_lft forever preferred_lft forever
檢視ip
#ka1
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188
# ka2
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200
ka1主機故障,測試VIP漂移至ka2主機
[root@centos8 ~]# systemctl stop keepalived
[root@centos8 ~]# hostname -I
172.31.0.28
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.188 172.31.0.200
恢復ka1主機
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200
模擬腦裂現象
# ka1
[root@centos8 ~]# iptables -A INPUT -s 172.31.0.48 -j DROP
[root@centos8 ~]# iptables -vnL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
12 480 DROP all -- * * 172.31.0.48 0.0.0.0/0
...
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188 172.31.0.200
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200
# ka2
[root@centos8 ~]# iptables -A INPUT -s 172.31.0.28 -j REJECT
[root@centos8 ~]# iptables -vnL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
10 400 REJECT all -- * * 172.31.0.28 0.0.0.0/0 reject-with icmp-port-unreachable
...
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200 172.31.0.188
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188 172.31.0.200
總結腦裂現象:當ka1新增錯誤的iptables規則後查詢ip就會看到有三個ip地址;ka2查詢還是原來的兩個ip地址,同時如果ka2也添加了錯誤的iptables規則後,查詢ip就會看到是三個ip地址,首先backup收不到master的vrrp通訊心跳檢測,backup就會把虛擬vip搶過來,但實際master還存在的,只是拒絕了backup請求響應從而造成腦裂現象
實現IPVS的高可用性
虛擬伺服器配置結構
virtual_server IP port {
...
real_server {
...
}
real_server {
...
}
…
}
virtual server (虛擬伺服器)的定義格式
virtual_server IP port #定義虛擬主機IP地址及其埠
virtual_server fwmark int #ipvs的防火牆打標,實現基於防火牆的負載均衡叢集
virtual_server group string #使用虛擬伺服器組
虛擬伺服器組
將多個虛擬伺服器定義成一個組,統一對外服務,如:http和https定義成一個虛擬伺服器組
#參考文件:/usr/share/doc/keepalived/keepalived.conf.virtual_server_group
virtual_server_group <STRING> {
# Virtual IP Address and Port
<IPADDR> <PORT>
<IPADDR> <PORT>
...
# <IPADDR RANGE> has the form
...
虛擬伺服器配置
virtual_server IP port { #VIP和PORT
delay_loop <INT> #檢查後端伺服器的時間間隔
lb_algo rr|wrr|lc|wlc|lblc|sh|dh #定義排程方法
lb_kind NAT|DR|TUN #叢集的型別,注意要大寫
persistence_timeout <INT> #持久連線時長
protocol TCP|UDP|SCTP #指定服務協議,一般為TCP
sorry_server <IPADDR> <PORT> #所有RS故障時,備用伺服器地址
real_server <IPADDR> <PORT> { #RS的IP和PORT
weight <INT> #RS權重
notify_up <STRING>|<QUOTED-STRING> #RS上線通知指令碼
notify_down <STRING>|<QUOTED-STRING> #RS下線通知指令碼
HTTP_GET|SSL_GET|TCP_CHECK|SMTP_CHECK|MISC_CHECK { ... } #定義當前主機健康狀態檢測方法
}
}
#注意:括號必須分行寫,兩個括號寫在同一行,如: }} 會出錯
應用層監測
應用層檢測:HTTP_GET|SSL_GET
HTTP_GET|SSL_GET {
url {
path <URL_PATH> #定義要監控的URL
status_code <INT> #判斷上述檢測機制為健康狀態的響應碼,一般為 200
}
connect_timeout <INTEGER> #客戶端請求的超時時長, 相當於haproxy的timeout server
nb_get_retry <INT> #重試次數
delay_before_retry <INT> #重試之前的延遲時長
connect_ip <IP ADDRESS> #向當前RS哪個IP地址發起健康狀態檢測請求
connect_port <PORT> #向當前RS的哪個PORT發起健康狀態檢測請求
bindto <IP ADDRESS> #向當前RS發出健康狀態檢測請求時使用的源地址
bind_port <PORT> #向當前RS發出健康狀態檢測請求時使用的源埠
}
範例:
virtual_server 172.31.0.188 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.17 80 {
weight 1
HTTP_GET {
url {
path /monitor.html
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 172.31.0.27 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
}
TCP監測
傳輸層檢測:TCP_CHECK
TCP_CHECK {
connect_ip <IP ADDRESS> #向當前RS的哪個IP地址發起健康狀態檢測請求
connect_port <PORT> #向當前RS的哪個PORT發起健康狀態檢測請求
bindto <IP ADDRESS> #發出健康狀態檢測請求時使用的源地址
bind_port <PORT> #發出健康狀態檢測請求時使用的源埠
connect_timeout <INTEGER> #客戶端請求的超時時長, 等於haproxy的timeout server
}
範例:
virtual_server 172.31.0.188 80 {
delay_loop 6
lb_algo wrr
lb_kind DR
#persistence_timeout 120 #會話保持時間
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.17 80 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 172.31.0.27 80 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
實戰案例1:實現單主的 LVS-DR 模式
準備web伺服器並使用指令碼繫結VIP至web伺服器lo網絡卡
#準備兩臺後端RS主機
[root@rs1 ~]# cat lvs_dr_rs.sh
#!/bin/bash
#Author:xuanlv
#Date:2021-06-13
vip=172.31.0.188
mask='255.255.255.255'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "<h1>`hostname`</h1>" > /var/www/html/index.html
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ifconfig $dev $vip netmask $mask #broadcast $vip up #route add -host $vip dev $dev
echo "The RS Server is Ready!"
;;
stop)
ifconfig $dev down
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo "The RS Server is Canceled!"
;;
*)
echo "Usage: $(basename $0) start|stop"
exit 1
;;
esac
# 執行
[root@rs1 ~]# bash lvs_dr_rs.sh start
[root@rs1 ~]# bash lvs_dr_rs.sh start
#測試直接訪問兩臺RS
配置keepalived
#ka1節點的配置
[root@ka1-centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka1.longxuan.vip
vrrp_mcast_group4 224.0.100.100
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 66
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
virtual_server 172.31.0.188 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.27 80 {
weight 1
HTTP_GET { #應用層檢測
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 172.31.0.37 80 {
weight 1
TCP_CHECK { #另一臺主機使用TCP檢測
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
#ka2節點的配置,配置和ka1基本相同,只需修改三行
[root@ka2-centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka2.longxuan.vip #修改此行
vrrp_mcast_group4 224.0.100.100
}
vrrp_instance VI_1 {
state BACKUP #修改此行
interface eth0
virtual_router_id 66
priority 80 #修改此行
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
virtual_server 172.31.0.188 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.27 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 172.31.0.37 80 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
訪問測試結果
root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188
<h1>rs1 web</h1>
安裝ipvsadm
[root@ka1 ~]# yum install ipvsadm -y
# 檢視ipvsadm
[root@ka1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.31.0.188:80 rr
-> 172.31.0.27:80 Route 1 0 0
-> 172.31.0.37:80 Route 1 0 0
模擬故障
#第一臺RS1故障,自動切換至RS2,這裡一開始訪問還會去RS1檢測,經過檢測幾次發現真的檢測不了,就不會往RS1排程,直接排程RS2
root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>403 Forbidden</title>
</head><body>
<h1>Forbidden</h1>
<p>You don't have permission to access /index.html
on this server.</p>
</body></html>
root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188
#檢查ipvsadm還有一臺可以排程,說明機器不能排程後,ipvsadm會自動剔除掉
[root@ka2 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.31.0.188:80 rr
-> 172.31.0.37:80 Route 1 0 0
當後端兩臺RS都故障時,就會啟用Sorry Server這臺服務,前提是這臺在ka1必須安裝有web伺服器,如果後端RS其中一臺恢復了也不會訪問Sorry Server服務
root@long:~# curl 172.31.0.188
Sorry Server
實戰案例2:實現雙主的 LVS-DR 模式
[root@ka1-centos8 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka1 #另一個節點為ka2
vrrp_mcast_group4 224.0.100.10
}
vrrp_instance VI_1 {
state MASTER #在另一個結點上為BACKUP
interface eth0
virtual_router_id 66
priority 100 #在另一個結點上為80
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1 #指定VIP
}
}
vrrp_instance VI_2 {
state BACKUP #在另一個結點上為MASTER
interface eth0
virtual_router_id 88
priority 80 #在另一個結點上為100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.200/24 dev eth0 label eth0:2 #指定VIP2
}
}
virtual_server 172.31.0.17 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.27 80 { #指定RS1地址
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 172.31.0.37 80 { #指定RS2地址
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
virtual_server 172.31.0.200 80 { #指定VIP2
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.27 80 { #指定RS3地址
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 172.31.0.37 80 { #指定RS4地址
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}