6、部署Haproxy
6.1 所有控制節點安裝Haproxy，並保持相同的配置，對於配置檔案，建議根據實際情況進行優化。
6.1.1安裝並配置haproxy

[root@controller1:/root]# yum -y install haproxy
[root@controller2:/root]# yum -y install haproxy
[root@controller3:/root]# yum -y install haproxy
[root@controller1:/root]# vim /etc/rsyslog.d/haproxy.conf

##配置HAProxy的日誌

# vim 
 
/etc/rsyslog.conf
…
$ModLoad imudp
$UDPServerRun 514
…
local2.* /var/log/haproxy/haproxy.log
…

# mkdir -pv /var/log/haproxy/
mkdir: created directory ‘/var/log/haproxy/’

# systemctl restart rsyslog

[root@controller1:/root]# scp /etc/rsyslog.d/haproxy.conf controller2:/etc/rsyslog.d/
[root@controller1:/root]# scp /etc/rsyslog.d/haproxy.conf controller3:/etc/rsyslog.d/
[root@controller1:
 
/root]# systemctl enable haproxy.service
[root@controller1:/root]# systemctl restart rsyslog.service
[root@controller1:/root]# systemctl status rsyslog.service
[root@controller2:/root]# systemctl enable haproxy.service
[root@controller2:/root]# systemctl restart rsyslog.service
[root@controller2:/root]# systemctl status rsyslog.service
[root@controller3:
 
/root]# systemctl enable haproxy.service
[root@controller3:/root]# systemctl restart rsyslog.service
[root@controller3:/root]# systemctl status rsyslog.service


[root@controller1:/root]# vim /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Example configuration for a possible web application. See the
# full configuration options online.
#
# http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local3

chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 40000
user haproxy
group haproxy
daemon

# turn on stats unix socket
stats socket /var/lib/haproxy/stats
spread-checks 3
tune.bufsize 32768
tune.maxrewrite 1024
tune.ssl.default-dh-param 2048
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
option httplog
option dontlognull
option tcplog
option splice-auto
option http-server-close
# option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 20s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
# timeout http-keep-alive 10s
timeout check 10s
maxconn 8000

listen stats
bind 0.0.0.0:8789
mode http
stats enable
stats uri /
stats realm Haproxy\ Statistics
stats auth admin:admin
stats refresh 15s
stats show-node
stats show-legends
stats hide-version
listen dashboard_cluster
bind 192.168.110.120:8080
balance source
option tcpka
option httpchk
option tcplog
server controller1 192.168.110.121:80 check inter 2000 rise 2 fall 5
server controller2 192.168.110.122:80 check inter 2000 rise 2 fall 5
server controller3 192.168.110.123:80 check inter 2000 rise 2 fall 5
listen galera_cluster
bind 192.168.110.120:3307
balance source
hash-type consistent
mode tcp
option tcplog
option clitcpka
option httpchk
timeout client 28801s
timeout server 28801s
server controller1 192.168.110.121:3306 check port 9200 inter 20s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:3306 check port 9200 inter 20s fastinter 2s downinter 2s rise 3 fall 3 backup
server controller3 192.168.110.123:3306 check port 9200 inter 20s fastinter 2s downinter 2s rise 3 fall 3 backup
listen mq_cluster
bind 192.168.110.120:5672 tcp-ut 5s
mode tcp
option tcpka
balance roundrobin
server controller1 192.168.110.121:5672 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:5672 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:5672 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen keystone_admin_cluster
bind 192.168.110.120:5001
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httplog
option httpclose
option forwardfor
server controller1 192.168.110.121:5000 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:5000 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:5000 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen keystone_public_internal_cluster
bind 192.168.110.120:5001
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httplog
option httpclose
option forwardfor
server controller1 192.168.110.121:5000 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:5000 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:5000 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen glance_registry_cluster
bind 192.168.110.120:9192
timeout server 30m
server controller1 192.168.110.121:9191 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:9191 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:9191 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen glance_api_cluster
bind 192.168.110.120:9293
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httpchk /version
option httplog
option httpclose
timeout server 30m
server controller1 192.168.110.121:9292 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:9292 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:9292 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen nova_ec2_api_cluster
bind 192.168.110.120:9773
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httpchk
option httplog
option httpclose
timeout server 600s
server controller1 192.168.110.121:8773 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:8773 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:8773 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen nova_compute_api_cluster
bind 192.168.110.120:9774
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httpchk
option httplog
option httpclose
timeout server 600s
server controller1 192.168.110.121:8774 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:8774 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:8774 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen nova_metadate_api_cluster
bind 192.168.110.120:9775
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httpchk
option httplog
option httpclose
timeout server 600s
server controller1 192.168.110.121:8775 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:8775 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:8775 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen nova_vncproxy_cluster
bind 192.168.110.120:6081
http-request set-header X-Forwarded-Proto https if { ssl_fc }
server controller1 192.168.110.121:6080 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:6080 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:6080 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen nova_placement_cluster
bind 192.168.110.120:9778
http-request set-header X-Forwarded-Proto https if { ssl_fc }
server controller1 192.168.110.121:8778 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:8778 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:8778 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen neutron_api_cluster
bind 192.168.110.120:9997
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httpchk
option httplog
option httpclose
server controller1 192.168.110.121:9696 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:9696 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:9696 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen cinder_api_cluster
bind 192.168.110.120:9776
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httpchk
option httplog
option httpclose
server controller1 192.168.110.121:8776 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:8776 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:8776 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
listen ceilometer_api_cluster
bind 192.168.110.120:9777
http-request set-header X-Forwarded-Proto https if { ssl_fc }
option httplog
option httpclose
server controller1 192.168.110.121:8777 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller2 192.168.110.122:8777 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
server controller3 192.168.110.123:8777 check inter 10s fastinter 2s downinter 2s rise 3 fall 3
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
#frontend main *:5000
# acl url_static path_beg -i /static /images /javascript /stylesheets
# acl url_static path_end -i .jpg .gif .png .css .js

# use_backend static if url_static
# default_backend app

#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
#backend static
# balance roundrobin
# server static 127.0.0.1:4331 check

#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
#backend app
# balance roundrobin
# server app1 127.0.0.1:5001 check
# server app2 127.0.0.1:5002 check
# server app3 127.0.0.1:5003 check
# server app4 127.0.0.1:5004 check

[root@controller1:/root]# scp /etc/haproxy/haproxy.cfg controller2:/etc/haproxy/
[root@controller1:/root]# scp /etc/haproxy/haproxy.cfg controller3:/etc/haproxy/

 

6.2 配置Haproxy能監控Galera資料庫叢集
​ 在控制節點三臺MariaDB上執行下列操作

[root@controller$:/root]# mysql 
MariaDB [(none)]> use mysql;
MariaDB [mysql]> grant process on *.* to 'clustercheckuser'@'localhost' identified by 'clustercheckpassword!';
MariaDB [mysql]> grant process on *.* to 'clustercheckuser'@'%' identified by 'clustercheckpassword!';
MariaDB [mysql]> flush privileges;
MariaDB [mysql]> exit
[root@controller$:/root]# cat <<EOF> /etc/sysconfig/clustercheck
MYSQL_USERNAME="clustercheckuser" 
MYSQL_PASSWORD="clustercheckpassword!"
MYSQL_HOST="localhost"
MYSQL_PORT="3306"
EOF
[root@controller$:/root]# vim /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_syncookies = 1
fs.file-max = 655350
net.ipv4.ip_local_port_range = 1025 65000

[root@controller$:/root]# sysctl -p


//注：重啟systemctl restart haproxy.service 後，192.168.110.120:3307端口才可以登陸
[root@controller3:/root]# mysql -h192.168.110.120 -P3307 -uroot -p"123456"
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 54
Server version: 10.4.11-MariaDB MariaDB Server
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> grant process on *.* to 'clustercheckuser'@'localhost' identified by 'clustercheckpassword!';
Query OK, 0 rows affected (0.108 sec)
MariaDB [(none)]> grant process on *.* to 'clustercheckuser'@'%' identified by 'clustercheckpassword!';
Query OK, 0 rows affected (0.086 sec)
MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.091 sec)
MariaDB [(none)]> exit
Bye

 

6.3 開啟haproxy,並自啟動（controller1、controller2、controller3）

systemctl restart haproxy.service
systemctl enable haproxy
systemctl status haproxy


[root@controller1:/root]# clustercheck
HTTP/1.1 200 OK
Content-Type: text/plain
Connection: close
Content-Length: 40
Percona XtraDB Cluster Node is synced.

[root@controller2:/root]# clustercheck
HTTP/1.1 200 OK
Content-Type: text/plain
Connection: close
Content-Length: 40
Percona XtraDB Cluster Node is synced.

[root@controller3:/root]# clustercheck
HTTP/1.1 200 OK
Content-Type: text/plain
Connection: close
Content-Length: 40
Percona XtraDB Cluster Node is synced.

 

6.4 登陸http://192.168.110.120:8789/ 檢測狀態 預設賬戶:admin 密碼:admin