五、LVS+Keepalive+nginx實驗

阿新 • • 發佈：2020-09-17

一、實驗環境

[root@node1 ~]# uname -r
3.10.0-862.el7.x86_64
[root@node1 ~]# cat /etc/redhat-release 
CentOS Linux release 7.5.1804 (Core) 

lvs+keepalived master:inode1: 192.168.31.101
lvs+keepalived backup:inode2: 192.168.31.102
nginx01:inode3：192.168.31.103----頁面資訊 www.ywx3.com
nginx02:inode4：192.168.31.104----頁面資訊 www.ywx4.com

VIP: 
192.168.31.111

二、LVS+Keepalived+nginx的部署

1、nginx部署

nginx01(inode3)和nginx02(inode4)

yum install -y nginx
systemctl nginx start

#nginx01的頁面資訊
echo 'www.ywx3.com' > /usr/local/nginx/html/index.html

#nginx02的頁面資訊
echo 'www.ywx4.com' > /usr/local/nginx/html/index.html

[root@node3 ~]# curl 192.168.31.103
www.ywx3.com

[root@node4 ~]# curl 192.168.31.104
www.ywx4.com

2、部署lvs+keepalived

inode1和inode2

yum install -y ipvsadm keepalived

3、編寫lvs+keepalived的配置檔案

lvs+keepalived master

! Configuration File for keepalived
global_defs {
   notification_email {
      yaowangxi@163.com
   }
   notification_email_from yaowangxi@163.com
   smtp_server 127.0.0.1 
   smtp_connect_timeout  
30
   router_id LVS_Keepalived_Master
}
# VIP1
vrrp_instance VI_1 {
    state  MASTER  
    interface  ens160
    lvs_sync_daemon_inteface ens160
    virtual_router_id 51
    priority 100 
    advert_int 5
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.111   
    }
}
virtual_server 192.168.31.111 80 {
    delay_loop 6    
    lb_algo rr    
    lb_kind DR   
    persistence_timeout  60   
    protocol TCP     
    real_server 192.168.31.103 80 {
        weight 100        
        TCP_CHECK {
        connect_timeout 10 
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.31.104 80 {
        weight 100
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
}

lvs+keepalived backup

! Configuration File for keepalived
global_defs {
   notification_email {
      yaowangxi@163.com
   }
   notification_email_from yaowangxi@163.com
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30
   router_id LVS_Keepalived_Backup
}
# VIP1
vrrp_instance VI_1 {
    state  BACKUP 
    interface  ens160
    lvs_sync_daemon_inteface ens160
    virtual_router_id 51
    priority 90 
    advert_int 5
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.111   
    }
}
virtual_server 192.168.31.111 80 {
    delay_loop 6    
    lb_algo rr    
    lb_kind DR   
    persistence_timeout  60   
    protocol TCP     
    real_server 192.168.31.103 80 {
        weight 1        
        TCP_CHECK {
        connect_timeout 10 
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.31.104 80 {
        weight 1
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
}

4、配置nginx01和nginx02上的VIP並抑制arp廣播

配置VIP，nginx01(inode3)和nginx02(inode4)操作一樣

cd /etc/sysconfig/network-scripts/
cp cp ifcfg-lo ifcfg-lo:0
vim ifcfg-lo:0

DEVICE=lo:0
IPADDR=192.168.31.111
NETMASK=255.255.255.255
ONBOOT=yes
NAME=loopback

ifup ifcfg-lo:0

nginx01(inode3)
[root@node3 network-scripts]# ifconfig|sed -n '/lo:0/,$p'
lo:0: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 192.168.31.111  netmask 255.255.255.255
        loop  txqueuelen 1000  (Local Loopback)

nginx02(inode4)
[root@node4 network-scripts]#  ifconfig|sed -n '/lo:0/,$p'
lo:0: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 192.168.31.111  netmask 255.255.255.255
        loop  txqueuelen 1000  (Local Loopback)

配置arp抑制nginx01(inode3)和nginx02(inode4)操作一樣

echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p

5、啟動keepalived

lvs+keepalived master(inode1)和lvs+keepalived backup(inode2)

systemctl start keepalived

6、檢視VIP地址

VIP地址在inode1上
[root@node1 ~]# ip addr list|grep -w "192.168.31.111"
    inet 192.168.31.111/32 scope global ens160

7、檢視ipvsadm並訪問vip

在inode1上檢視
[root@node1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.31.111:80 rr persistent 60
  -> 192.168.31.103:80            Route   100    0          0         
  -> 192.168.31.104:80            Route   100    0          0

[root@node3 network-scripts]# curl 192.168.31.111
www.ywx3.com

[root@node4 network-scripts]# curl 192.168.31.111
www.ywx4.com

8、關閉nginx01,檢視ipvsadm（inode1上）

inode3
[root@node3 ~]# nginx -s stop
[root@node3 ~]# ps -ef |grep 'nginx'
root     17695 17147  0 08:05 pts/0    00:00:00 grep --color=auto nginx

inode1
[root@node1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.31.111:80 rr persistent 60
  -> 192.168.31.104:80            Route   100    0          1  
#RS nginx01(inode3)192.168.31.103已經被移除

9、恢復nginx01,inode3會自動新增進ipvsadm

nginx01(inode3)
[root@node3 ~]# nginx
[root@node3 ~]# ps -ef |grep 'nginx'
root     17699     1  0 08:07 ?        00:00:00 nginx: master process nginx
nginx    17700 17699  0 08:07 ?        00:00:00 nginx: worker process
root     17702 17147  0 08:07 pts/0    00:00:00 grep --color=auto nginx


inode1
[root@node1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.31.111:80 rr persistent 60
  -> 192.168.31.103:80            Route   1      0          0         
  -> 192.168.31.104:80            Route   100    0          0

10、關閉inode1的keepalived,VIP會自動漂移到inode2上

inode1
[root@node1 ~]# ip addr list|grep -w '192.168.31.111'
    inet 192.168.31.111/32 scope global ens160
[root@node1 ~]# systemctl stop keepalived
[root@node1 ~]# ip addr list|grep -w '192.168.31.111'
[root@node1 ~]# 

inode2
[root@node2 ~]# ip addr list|grep -w '192.168.31.111'
    inet 192.168.31.111/32 scope global ens160
[root@node2 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.31.111:80 rr persistent 60
  -> 192.168.31.103:80            Route   1      0          0         
  -> 192.168.31.104:80            Route   1      0          0

三、lvs+keepalived的配置檔案解析

#keepalived部分
global_defs {
   notification_email {
         yaowangxi@163.com
   }
   notification_email_from 441520481@qq.com
   smtp_server 192.168.80.1
   smtp_connection_timeout 30
   router_id LVS_DEVEL  # 設定lvs的id，在一個網路內應該是唯一的
}
vrrp_instance VI_1 {
    state MASTER #指定Keepalived的角色，MASTER為主，BACKUP為備 記得大寫
    interface eno160  #網絡卡id 不同的電腦網絡卡id會有區別 可以使用:ip a檢視
    virtual_router_id 51  #虛擬路由編號，主備要一致
    priority 100  #定義優先順序，數字越大，優先順序越高，主DR必須大於備用DR
    advert_int 1  #檢查間隔，預設為1s
    authentication {   #這裡配置的密碼最多為8位，主備要一致，否則無法正常通訊
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.111  #定義虛擬IP(VIP)為192.168.31.111，可多設，每行一個
    }
}
#LVS部分
# 定義對外提供服務的LVS的VIP以及port
virtual_server 192.168.31.111 80 {
    delay_loop 6 # 設定健康檢查時間，單位是秒
    lb_algo rr # 設定負載排程的演算法為wlc
    lb_kind DR # 設定LVS實現負載的機制，有NAT、TUN、DR三個模式
    nat_mask 255.255.255.0
    persistence_timeout 0
    protocol TCP
    real_server 192.168.31.103 80 {  # 指定real server1的IP地址
        weight 1   # 配置節點權值，數字越大權重越高
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.31.104 80 {  # 指定real server2的IP地址
        weight 1  # 配置節點權值，數字越大權重越高
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
     }
}

四、realserver指令碼檔案

#虛擬的vip 根據自己的實際情況定義
SNS_VIP=192.168.31.111
/etc/rc.d/init.d/functions
case "$1" in
start)
       ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
       /sbin/route add -host $SNS_VIP dev lo:0
       echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
       echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
       echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
       echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
       sysctl -p >/dev/null 2>&1
       echo "RealServer Start OK"
       ;;
stop)
       ifconfig lo:0 down
       route del $SNS_VIP >/dev/null 2>&1
       echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
       echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
       echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
       echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
       echo "RealServer Stoped"
       ;;
*)
       echo "Usage: $0 {start|stop}"
       exit 1
esac
exit 0