1. 程式人生 > 其它 >高可用keepalived之進階

高可用keepalived之進階

實現 master/master 的 Keepalived 雙主架構

master/slave的單主架構,同一時間只有一個Keepalived對外提供服務,此主機繁忙,而另一臺主機卻
很空閒,利用率低下,可以使用master/master的雙主架構,解決此問題。
master/master 的雙主架構:
即將兩個或以上VIP分別執行在不同的keepalived伺服器,以實現伺服器並行提供web訪問的目的,提高
伺服器資源利用率

#ha1主機配置
[root@ka1-centos8 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
    notification_email {
        [email protected]
    }
    notification_email_from keepalived@localhost
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id ka1.longxuan.vip
    vrrp_mcast_group4 224.8.8.188
}
vrrp_instance N520 {
    state MASTER #在另一個主機上為BACKUP
    interface eth0
    virtual_router_id 66 #每個vrrp_instance唯一
    priority 100 #在另一個主機上為80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188 dev eth0 label eth0:1 #指定vrrp_instance各自的VIP
    }
}
vrrp_instance N521 { #新增 VI_2 例項
    state BACKUP #在另一個主機上為MASTER
    interface eth0
    virtual_router_id 88 #每個vrrp_instance唯一
    priority 80 #在另一個主機上為100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 12345678
    }
    virtual_ipaddress {
        172.31.0.200 dev eth0 label eth0:2 #指定vrrp_instance各自的VIP
    }
}

#ka2主機配置,和ka1配置只需五行不同
[root@ka2-centos8 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
    notification_email {
        [email protected]
    }
    notification_email_from keepalived@localhost
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id ka2.longxuan.vip #修改此行
    vrrp_mcast_group4 224.8.8.100
}
vrrp_instance N520 {
    state BACKUP #此修改行為BACKUP
    interface eth0
    virtual_router_id 66
    priority 80 #此修改行為80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188 dev eth0 label eth0:1
    }
}
vrrp_instance N521 {
    state MASTER #修改此行為MASTER
    interface eth0
    virtual_router_id 88
    priority 100 #修改此行為100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 12345678
    }
    virtual_ipaddress {
        172.31.0.200 dev eth0 label eth0:2
    }
}

實戰案例:利用子配置檔案實現master/master的Keepalived雙主架構

範例:ka1配置

[root@centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen
   }
   notification_email_from [email protected]
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}
include /etc/keepalived/conf.d/*.conf

# 建立子目錄
[root@centos8 ~]# mkdir /etc/keepalived/conf.d/ -p

# 專案1-master
[root@centos8 ~]# cat /etc/keepalived/conf.d/n520.conf
vrrp_instance n520 {
    state MASTER
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    #nopreempt
    #preempt_delay 10
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188 dev eth0 label eth0:1
    }
    unicast_src_ip 172.31.0.28
    unicast_peer{
       172.31.0.48
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

# 專案2-backup
[root@centos8 ~]# vim /etc/keepalived/conf.d/n521.conf
vrrp_instance n521 {
    state BACKUP
    interface eth0
    virtual_router_id 88
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 12345678
    }
    virtual_ipaddress {
        172.31.0.200 dev eth0 label eth0:2
    }
    unicast_src_ip 172.31.0.28
    unicast_peer{
       172.31.0.48
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

# 重啟
[root@centos8 ~]# systemctl restart keepalived
檢查ip

[root@centos8 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:0c:29:ac:f5:a4 brd ff:ff:ff:ff:ff:ff
    inet 172.31.0.28/16 brd 172.31.255.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 172.31.0.188/32 scope global eth0:1
       valid_lft forever preferred_lft forever
    inet 172.31.0.200/32 scope global eth0:2

範例:ka2配置

[root@centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen
   }
   notification_email_from [email protected]
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}
include /etc/keepalived/conf.d/*.conf

# 建立子目錄
[root@centos8 ~]# mkdir /etc/keepalived/conf.d/ -p

# 專案1-backup
[root@centos8 ~]# cat /etc/keepalived/conf.d/n520.conf
vrrp_instance n520 {
    state BACKUP
    interface eth0
    virtual_router_id 66
    priority 80
    advert_int 1
    #nopreempt
    #preempt_delay 10   #如果設定延遲搶佔模式,需要都是BACKUP,不然會造成腦裂
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188 dev eth0 label eth0:1
    }
    unicast_src_ip 172.31.0.48
    unicast_peer{
       172.31.0.28
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

# 專案2-master
[root@centos8 ~]# cat //etc/keepalived/conf.d/n521.conf
vrrp_instance n521 {
    state MASTER
    interface eth0
    virtual_router_id 88
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 12345678
    }
    virtual_ipaddress {
        172.31.0.200 dev eth0 label eth0:2
    }
    unicast_src_ip 172.31.0.48
    unicast_peer{
       172.31.0.28
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

# 重啟keepalived
[root@centos8 ~]# systemctl restart keepalived

# 檢視ip
[root@centos8 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:0c:29:16:9a:81 brd ff:ff:ff:ff:ff:ff
    inet 172.31.0.48/16 brd 172.31.255.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 172.31.0.200/32 scope global eth0:2
       valid_lft forever preferred_lft forever

檢視ip

#ka1
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188

# ka2
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200

ka1主機故障,測試VIP漂移至ka2主機

[root@centos8 ~]# systemctl stop keepalived

[root@centos8 ~]# hostname -I
172.31.0.28

[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.188 172.31.0.200 

恢復ka1主機

[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188

[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200 

模擬腦裂現象

# ka1
[root@centos8 ~]# iptables -A INPUT -s 172.31.0.48 -j DROP
[root@centos8 ~]# iptables -vnL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination
   12   480 DROP       all  --  *      *       172.31.0.48          0.0.0.0/0
...

[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188 172.31.0.200

[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200 

# ka2
[root@centos8 ~]# iptables -A INPUT -s 172.31.0.28 -j REJECT
[root@centos8 ~]# iptables -vnL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination
   10   400 REJECT     all  --  *      *       172.31.0.28          0.0.0.0/0            reject-with icmp-port-unreachable
...

[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200 172.31.0.188 

[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188 172.31.0.200

總結腦裂現象:當ka1新增錯誤的iptables規則後查詢ip就會看到有三個ip地址;ka2查詢還是原來的兩個ip地址,同時如果ka2也添加了錯誤的iptables規則後,查詢ip就會看到是三個ip地址,首先backup收不到master的vrrp通訊心跳檢測,backup就會把虛擬vip搶過來,但實際master還存在的,只是拒絕了backup請求響應從而造成腦裂現象

實現IPVS的高可用性

虛擬伺服器配置結構

virtual_server IP port {
   ...
   real_server {
      ...
   }
   real_server {
     ...
   }
  …
}

virtual server (虛擬伺服器)的定義格式

virtual_server IP port #定義虛擬主機IP地址及其埠
virtual_server fwmark int #ipvs的防火牆打標,實現基於防火牆的負載均衡叢集
virtual_server group string #使用虛擬伺服器組

虛擬伺服器組

將多個虛擬伺服器定義成一個組,統一對外服務,如:http和https定義成一個虛擬伺服器組

#參考文件:/usr/share/doc/keepalived/keepalived.conf.virtual_server_group
virtual_server_group <STRING> {
# Virtual IP Address and Port
<IPADDR> <PORT>
<IPADDR> <PORT>
...
# <IPADDR RANGE> has the form
...

虛擬伺服器配置

virtual_server IP port { #VIP和PORT
    delay_loop <INT> #檢查後端伺服器的時間間隔
    lb_algo rr|wrr|lc|wlc|lblc|sh|dh #定義排程方法
    lb_kind NAT|DR|TUN #叢集的型別,注意要大寫
    persistence_timeout <INT> #持久連線時長
    protocol TCP|UDP|SCTP #指定服務協議,一般為TCP
    sorry_server <IPADDR> <PORT> #所有RS故障時,備用伺服器地址
    real_server <IPADDR> <PORT> { #RS的IP和PORT
        weight <INT> #RS權重
        notify_up <STRING>|<QUOTED-STRING> #RS上線通知指令碼
        notify_down <STRING>|<QUOTED-STRING> #RS下線通知指令碼
        HTTP_GET|SSL_GET|TCP_CHECK|SMTP_CHECK|MISC_CHECK { ... } #定義當前主機健康狀態檢測方法
    }
}
#注意:括號必須分行寫,兩個括號寫在同一行,如: }} 會出錯

應用層監測

應用層檢測:HTTP_GET|SSL_GET

HTTP_GET|SSL_GET {
   url {
      path <URL_PATH> #定義要監控的URL
      status_code <INT> #判斷上述檢測機制為健康狀態的響應碼,一般為 200
    }
    connect_timeout <INTEGER> #客戶端請求的超時時長, 相當於haproxy的timeout server
    nb_get_retry <INT> #重試次數
    delay_before_retry <INT> #重試之前的延遲時長
    connect_ip <IP ADDRESS> #向當前RS哪個IP地址發起健康狀態檢測請求
    connect_port <PORT> #向當前RS的哪個PORT發起健康狀態檢測請求
    bindto <IP ADDRESS> #向當前RS發出健康狀態檢測請求時使用的源地址
    bind_port <PORT> #向當前RS發出健康狀態檢測請求時使用的源埠
}

範例:

virtual_server 172.31.0.188 80 {
    delay_loop 3
    lb_algo rr
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 172.31.0.17 80 {
        weight 1
        HTTP_GET {
           url {
               path /monitor.html
               status_code 200
           }
           connect_timeout 1
           nb_get_retry 3
           delay_before_retry 1
        }
    }
    real_server 172.31.0.27 80 {
        weight 1
        HTTP_GET {
           url {
               path /
               status_code 200
           }
           connect_timeout 1
           nb_get_retry 3
           delay_before_retry 1
        }
    }
}

TCP監測

傳輸層檢測:TCP_CHECK

TCP_CHECK {
    connect_ip <IP ADDRESS> #向當前RS的哪個IP地址發起健康狀態檢測請求
    connect_port <PORT> #向當前RS的哪個PORT發起健康狀態檢測請求
    bindto <IP ADDRESS> #發出健康狀態檢測請求時使用的源地址
    bind_port <PORT> #發出健康狀態檢測請求時使用的源埠
    connect_timeout <INTEGER> #客戶端請求的超時時長, 等於haproxy的timeout server
}

範例:

virtual_server 172.31.0.188 80 {
    delay_loop 6
    lb_algo wrr
    lb_kind DR
    #persistence_timeout 120 #會話保持時間
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 172.31.0.17 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 5
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
            }
    }
    real_server 172.31.0.27 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 5
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }
}

實戰案例1:實現單主的 LVS-DR 模式

準備web伺服器並使用指令碼繫結VIP至web伺服器lo網絡卡

#準備兩臺後端RS主機
[root@rs1 ~]# cat lvs_dr_rs.sh
#!/bin/bash
#Author:xuanlv
#Date:2021-06-13
vip=172.31.0.188
mask='255.255.255.255'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "<h1>`hostname`</h1>" > /var/www/html/index.html
case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ifconfig $dev $vip netmask $mask #broadcast $vip up #route add -host $vip dev $dev
    echo "The RS Server is Ready!"
    ;;
stop)
    ifconfig $dev down
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "The RS Server is Canceled!"
    ;;
*)
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
esac

# 執行
[root@rs1 ~]# bash lvs_dr_rs.sh start
[root@rs1 ~]# bash lvs_dr_rs.sh start

#測試直接訪問兩臺RS

配置keepalived

#ka1節點的配置
[root@ka1-centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
   notification_email {
       root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id ka1.longxuan.vip
   vrrp_mcast_group4 224.0.100.100
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188 dev eth0 label eth0:1
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    }
    virtual_server 172.31.0.188 80 {
        delay_loop 3
        lb_algo rr
        lb_kind DR
        protocol TCP
        sorry_server 127.0.0.1 80
        real_server 172.31.0.27 80 {
            weight 1
            HTTP_GET { #應用層檢測
                url {
                   path /
                   status_code 200
                }
                connect_timeout 1
                nb_get_retry 3
                delay_before_retry 1
            }
        }
        real_server 172.31.0.37 80 {
            weight 1
            TCP_CHECK { #另一臺主機使用TCP檢測
                connect_timeout 5
                nb_get_retry 3
                delay_before_retry 3
                connect_port 80
            }
        }
}

#ka2節點的配置,配置和ka1基本相同,只需修改三行
[root@ka2-centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
    notification_email {
        root@localhost
    }
    notification_email_from keepalived@localhost
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id ka2.longxuan.vip #修改此行
    vrrp_mcast_group4 224.0.100.100
}
vrrp_instance VI_1 {
    state BACKUP #修改此行
    interface eth0
    virtual_router_id 66
    priority 80 #修改此行
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188 dev eth0 label eth0:1
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    }
virtual_server 172.31.0.188 80 {
    delay_loop 3
    lb_algo rr
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 172.31.0.27 80 {
        weight 1
        HTTP_GET {
            url {
               path /
               status_code 200
            }
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1
        }
    }
    real_server 172.31.0.37 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 5
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }
}

訪問測試結果

root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188
<h1>rs1 web</h1>

安裝ipvsadm

[root@ka1 ~]# yum install ipvsadm -y
# 檢視ipvsadm
[root@ka1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  172.31.0.188:80 rr
  -> 172.31.0.27:80               Route   1      0          0
  -> 172.31.0.37:80               Route   1      0          0

模擬故障

#第一臺RS1故障,自動切換至RS2,這裡一開始訪問還會去RS1檢測,經過檢測幾次發現真的檢測不了,就不會往RS1排程,直接排程RS2
root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>403 Forbidden</title>
</head><body>
<h1>Forbidden</h1>
<p>You don't have permission to access /index.html
on this server.</p>
</body></html>
root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188

#檢查ipvsadm還有一臺可以排程,說明機器不能排程後,ipvsadm會自動剔除掉
[root@ka2 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  172.31.0.188:80 rr
  -> 172.31.0.37:80               Route   1      0          0

當後端兩臺RS都故障時,就會啟用Sorry Server這臺服務,前提是這臺在ka1必須安裝有web伺服器,如果後端RS其中一臺恢復了也不會訪問Sorry Server服務

root@long:~# curl 172.31.0.188
Sorry Server

實戰案例2:實現雙主的 LVS-DR 模式

[root@ka1-centos8 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
    notification_email {
        root@localhost
    }
    notification_email_from keepalived@localhost
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id ka1     #另一個節點為ka2
    vrrp_mcast_group4 224.0.100.10
}
vrrp_instance VI_1 {
    state MASTER    #在另一個結點上為BACKUP
    interface eth0
    virtual_router_id 66
    priority 100   #在另一個結點上為80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188 dev eth0 label eth0:1 #指定VIP
    }
}
vrrp_instance VI_2 {
    state BACKUP #在另一個結點上為MASTER
    interface eth0
    virtual_router_id 88
    priority 80 #在另一個結點上為100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.200/24 dev eth0 label eth0:2 #指定VIP2
    }
}
virtual_server 172.31.0.17 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 172.31.0.27 80 { #指定RS1地址
        weight 1
        HTTP_GET {
            url {
               path /
               status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
    real_server 172.31.0.37 80 { #指定RS2地址
        weight 1
        HTTP_GET {
            url {
                path /
                status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
virtual_server 172.31.0.200 80 { #指定VIP2
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 172.31.0.27 80 { #指定RS3地址
        weight 1
        HTTP_GET {
            url {
                path /
                status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
    real_server 172.31.0.37 80 { #指定RS4地址
        weight 1
        HTTP_GET {
            url {
                path /
                status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}