PG-高可用patroni+etcd實驗環境部署
阿新 • • 發佈：2020-07-31
部署實施
Patroni + Etcd 方案
軟體版本規劃如下:
- 作業系統: rhel Linux 7.6
- 資料庫: PostgreSQL 12.2
- Python: Python 3.8.2
- Etcd: etcd-v3.3.22
- Patroni: patroni 1.6.5
部署規劃如下:
| 主機 | IP | 元件 | 備註 |
|---|---|---|---|
| pg1 | 192.168.10.190 | PostgreSQL、Patroni、Etcd | 主節點 |
| pg2 | 192.168.10.191 | PostgreSQL、Patroni、Etcd | 備節點1 |
| pg3 | 192.168.10.192 | PostgreSQL、Patroni、Etcd | 備節點2 |
軟體地址
#-- patroni https://github.com/zalando/patroni https://mirrors.aliyun.com/pypi/simple/patroni/ #-- etcd https://github.com/etcd-io/etcd/releases/tag/v3.4.10 https://github.com/etcd-io/etcd/releases/tag/v3.3.22 #-- python #-- 自簽證書工具 cfssl，cfssljson wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
環境準備
安裝系統依賴軟體包
yum -y install libffi-devel gcc gcc-c++ zlib zlib-devel readline-devel openssl-devel bzip2-devel sqlite-devel xz lzma xz-devel gdbm gdbm-devel tk tk-devel libffi libffi-devel ncurses ncurses-devel
配置EPEL yum源
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo yum -y install jq
配置sudo許可權
cat > /etc/sudoers.d/postgres <<-EOF
# postgres ALL=(ALL:ALL) NOPASSWD:ALL
postgres ALL=(ALL:ALL) NOPASSWD:/bin/systemctl stop postgresql12,/bin/systemctl start postgresql12,/bin/systemctl restart postgresql12,/bin/systemctl status postgresql12,/bin/systemctl daemon-reload,/bin/systemctl stop patroni,/bin/systemctl start patroni,/bin/systemctl restart patroni,/bin/systemctl status patroni,/bin/systemctl stop etcd,/bin/systemctl start etcd,/bin/systemctl restart etcd,/bin/systemctl status etcd
EOF
安裝python3
# wget -c https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tar.xz
./configure --prefix=/ups/app/postgresql/pgsql-12 --with-perl --with-tcl --with-python --with-openssl --with-pam --without-ldap --with-libxml --with-libxslt --with-systemd
make world
make install-world
配置動態庫
cat > /etc/ld.so.conf.d/python3.conf <<-EOF
/ups/app/python3/lib
EOF
# 載入動態庫
ldconfig -v | grep python3
# 檢查確認
ldd /ups/app/python3/bin/python3
安裝PG流複製(一主兩從)
編譯安裝軟體
mkdir build_dir && cd build_dir
../configure --prefix=/ups/app/postgresql/pgsql-12 --with-perl --with-tcl --with-python --with-openssl --with-pam --with-gssapi --with-icu --without-ldap --with-libxml --with-libxslt --with-systemd
make world
make install-world
主庫初始化
initdb -D ${PGDATA} -U postgres --locale=en_US.UTF8 -E UTF8
配置主庫引數配置
vi $PGDATA/postgresql.conf
cat >> $PGDATA/postgresql.conf <<-EOF
listen_addresses = '*'
port = 2020
wal_level = replica
archive_mode = on
#archive_command = 'test ! -f /ups/data/pgdata/12/arch/%f && cp %p /ups/data/pgdata/12/arch/%f'
#restore_command = 'cp /ups/data/pgdata/12/arch/%f %p'
archive_command = '/usr/bin/lz4 -q -z %p /ups/data/pgdata/12/arch/%f.lz4'
restore_command = '/usr/bin/lz4 -d /ups/data/pgdata/12/arch/%f.lz4 %p'
recovery_target_timeline='latest'
max_wal_senders = 10
wal_keep_segments = 64
hot_standby = on
hot_standby_feedback = on
full_page_writes = on
wal_log_hits = on
synchronous_commit = on
synchronous_standby_names = 'ANY 1 (*)'
archive_cleanup_command = 'pg_archivecleanup /ups/data/pgdata/12/arch %r'
primary_slot_name ='pgsql12_pg1'
EOF
檢查確認配置檔案
grep -Ev '^[[:space:]]|^#|^$' ${PGDATA}/postgresql.conf
sed -r '/^[ \t]*($|#)/d' $PGDATA/postgresql.conf
啟動主庫
pg_ctl start -D $PGDATA
配置自啟動服務
# 編輯服務檔案
cat > /usr/lib/systemd/system/postgresql12.service <<-EOF
# It's not recommended to modify this file in-place, because it will be
# overwritten during package upgrades. If you want to customize, the
# best way is to create a file "/etc/systemd/system/postgresql12.service",
# containing
# .include /usr/lib/systemd/system/postgresql12.service
# ...make your changes here...
# For more info about custom unit files, see
# http://fedoraproject.org/wiki/Systemd#How_do_I_customize_a_unit_file.2F_add_a_custom_unit_file.3F
# Note: changing PGDATA will typically require adjusting SELinux
# configuration as well.
# Note: do not use a PGDATA pathname containing spaces, or you will
# break postgresql-setup.
[Unit]
Description=PostgreSQL 12 database server
Documentation=https://www.postgresql.org/docs/12/static/
After=syslog.target
After=network.target
[Service]
Type=notify
User=postgres
Group=postgres
# Note: avoid inserting whitespace in these Environment= lines, or you may
# break postgresql-setup.
# Location of database directory
Environment=PGDATA=/ups/data/pgdata/12/pg_root
Environment=PGHOME=/ups/app/postgresql/pgsql-12
# Where to send early-startup messages from the server (before the logging
# options of postgresql.conf take effect)
# This is normally controlled by the global default set by systemd
# StandardOutput=syslog
# Disable OOM kill on the postmaster
OOMScoreAdjust=-1000
Environment=PG_OOM_ADJUST_FILE=/proc/self/oom_score_adj
Environment=PG_OOM_ADJUST_VALUE=0
# ExecStartPre=/ups/app/postgresql/pgsql-12/bin/postgresql-12-check-db-dir \${PGDATA}
ExecStart=/ups/app/postgresql/pgsql-12/bin/postmaster -D \${PGDATA}
ExecReload=/bin/kill -HUP \$MAINPID
KillMode=mixed
KillSignal=SIGINT
# Do not set any timeout value, so that systemd will not kill postmaster
# during crash recovery.
# 禁用超時邏輯,systemd的預設超時時長是 90 秒
TimeoutSec=0
[Install]
WantedBy=multi-user.target
EOF
禁止postgresql自啟動，通過patroni來管理postgresql
配置postgres使用者密碼
psql -U postgres -c "ALTER USER postgres WITH PASSWORD 'postgres';"
建立流複製賬號(pg1)
CREATE USER sync
REPLICATION
LOGIN
ENCRYPTED PASSWORD 'sync12345';
-- CONNECTION LIMIT 5
GRANT EXECUTE ON FUNCTION pg_read_binary_file(text) TO sync;
建立複製槽（pg1）patroni
-- 建立複製槽
select * from pg_create_physical_replication_slot('pgsql12_pg1');
select * from pg_create_physical_replication_slot('pgsql12_pg2');
select * from pg_create_physical_replication_slot('pgsql12_pg3');
-- 檢視
select * from pg_replication_slots;
-- 刪除複製槽
SELECT * FROM pg_drop_replication_slot('pgsql12_pg1');
配置客戶端認證檔案（pg_hba.conf) 所有伺服器
cat >> ${PGDATA}/pg_hba.conf <<EOF
host all all 192.168.10.0/24 md5
host replication sync 127.0.0.1/32 md5
host replication sync 192.168.10.190/24 md5
host replication sync 192.168.10.191/24 md5
host replication sync 192.168.10.192/24 md5
EOF
pg_basebackup同步資料時，自動部署到其它節點從庫
配置使用者口令檔案（$HOME/.pgpass） 所有伺服器
touch ~/.pgpass
chmod 0600 ~/.pgpass
cat > ~/.pgpass <<EOF
192.168.10.190:2020:replication:sync:sync12345
192.168.10.191:2020:replication:sync:sync12345
192.168.10.192:2020:replication:sync:sync12345
EOF
cat <<-EOF >~/.pgpass
192.168.10.190:2020:replication:sync:sync12345
192.168.10.191:2020:replication:sync:sync12345
192.168.10.192:2020:replication:sync:sync12345
EOF
備庫初始化
# pg2
pg_basebackup -h 192.168.10.190 -p 2020 -U sync -D ${PGDATA} -w -Fp -Xs -Pv -R
# pg3
pg_basebackup -h 192.168.10.190 -p 2020 -U sync -D ${PGDATA} -w -Fp -Xs -Pv -R
-R, --write-recovery-conf：write configuration for replication- 自動建立$PGDATA/standby.signal 標識檔案，且該檔案內容為空
- 自動在$PGDATA/postgresql.auto.conf 檔案中新增 primary_conninfo 引數資訊
修改備庫引數
在postgre.auto.conf 新增 application_name =slave1
primary_conninfo = 'application_name=pgsql12_pg2 user=sync passfile=''/home/postgres/.pgpass'' host=192.168.10.190 port=2020 sslmode=prefer sslcompression=0 gssencmode=prefer krbsrvname=postgres target_session_attrs=any'
primary_slot_name ='pgsql12_pg2'
啟動備庫
pg_ctl start -D $PGDATA
檢查同步
SELECT usename , application_name , client_addr, sync_state FROM pg_stat_replication;
-- 結果是f(false)則為主庫，t(true)為備庫
SELECT pg_is_in_recovery();
-- 檢查複製情況
SELECT pid, usename, client_addr, state, write_lag, flush_lag, replay_lag FROM pg_stat_replication;
SELECT pid, usename, client_addr, state, pg_wal_lsn_diff(pg_current_wal_lsn(), write_lsn) write_delay, pg_wal_lsn_diff(pg_current_wal_lsn(), flush_lsn) flush_delay, pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) replay_delay FROM pg_stat_replication;
自簽證書配置
安裝 CFSSL相關軟體
export WORK_DIR=/ups/app/cfssl
mkdir -p ${WORK_DIR} && cd ${WORK_DIR}
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O ${WORK_DIR}/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O ${WORK_DIR}/cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O ${WORK_DIR}/cfssl-certinfo
export WORK_DIR=/ups/app/cfssl
for value in $(ls -tr .); do
tmp=$(echo ${value} |awk -F"_" '{print $1}')
mv ${value} ${tmp}
chmod +x ${tmp}
ln -s ${WORK_DIR}/${tmp} /usr/local/bin/${tmp}
done
驗證 cfssl 的版本為 1.2或更高
檢視版本
[root@pg1 cfssl]# cfssl version
Version: 1.2.0
Revision: dev
Runtime: go1.6
[root@pg1 cfssl]#
工具命令語法
cfssl
Usage:
Available commands:
selfsign
print-defaults
revoke
certinfo
serve
version
gencert
gencrl
info
bundle
sign
genkey
ocspsign
ocspserve
ocspdump
ocsprefresh
scan
Top-level flags:
-allow_verification_with_non_compliant_keys
Allow a SignatureVerifier to use keys which are technically non-compliant with RFC6962.
-loglevel int
Log level (0 = DEBUG, 5 = FATAL) (default 1)
cfssljson
Usage of cfssljson:
-bare
the response from CFSSL is not wrapped in the API standard response
-f string
JSON input (default "-")
-stdout
output the response instead of saving to a file
建立 CA(Certificate Authority) 證書
先用 cfssl 命令生成包含預設配置的 ca-config.json 和 ca-csr.json 檔案
建立 CA 配置檔案（ca-config.json）
cd /ups/app/etcd/ssl
cfssl print-defaults config > ca-config.json
cfssl print-defaults csr > ca-csr.json
然後分別修改這兩個檔案為如下內容
ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"server": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
},
"client": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
}
}
}
}
-
ca-config.json：
可以定義多個 profiles，分別指定不同的過期時間、使用場景等引數；後續在簽名證書時使用某個 profile；
-
signing：
表示該證書可用於簽名其它證書；生成的 ca.pem 證書中 CA=TRUE；
-
server auth：
表示 Client 可以用該 CA 對 Server 提供的證書進行驗證；
-
client auth：
表示 Server 可以用該 CA 對 Client 提供的證書進行驗證；
建立 CA 證書籤名請求（ca-csr.json）
ca-csr.json
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Guangzhou",
"ST": "Guangdong"
}
]
}
-
CN(Common Name)：
- kube-apiserver 從證書中提取該欄位作為請求的使用者名稱（User Name）；瀏覽器使用該欄位驗證網站是否合法。一般寫都是域名
-
ST(State)：
- 州、省
-
L(Locality)：
- 地區、城市
-
O(Organization Name)：
- 組織名稱，公司名稱。kube-apiserver 從證書中提取該欄位作為請求使用者所屬的組（Group）
-
OU(Organization Unit Name):
- 組織單位名稱，公司部門名稱
前面2步可以通過以下方式建立CA證書
cat > /ups/app/etcd/ssl/ca-config.json <<-EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"server": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
},
"client": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
}
}
}
}
EOF
cat > /ups/app/etcd/ssl/ca-csr.json <<-EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GD",
"L": "GZ"
}
]
}
EOF
生成 CA 證書和私鑰
cd /ups/app/etcd/ssl
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
[root@pg1 ssl]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca
2020/07/29 20:01:32 [INFO] generating a new CA key and certificate from CSR
2020/07/29 20:01:32 [INFO] generate received request
2020/07/29 20:01:32 [INFO] received CSR
2020/07/29 20:01:32 [INFO] generating key: rsa-2048
2020/07/29 20:01:32 [INFO] encoded CSR
2020/07/29 20:01:32 [INFO] signed certificate with serial number 51387564996842455704424109245687617617608202805
[root@pg1 ssl]#
CA 有關證書列表如下：
[root@pg1 ssl]# tree
.
├── ca.csr
├── ca-key.pem
├── ca.pem
├── ca-config.json
└── ca-csr.json
建立 etcd 的TLS認證證書，生成 etcd 證書和私鑰
在 /ups/app/etcd/ssl 下新增檔案 etcd-csr.json，內容如下
cat > /ups/app/etcd/ssl/etcd-csr.json <<-EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.10.190",
"192.168.10.191",
"192.168.10.192"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GD",
"L": "GZ"
}
]
}
EOF
生成 etcd server證書
cd /ups/app/etcd/ssl
cfssl gencert -ca=/ups/app/etcd/ssl/ca.pem \
-ca-key=/ups/app/etcd/ssl/ca-key.pem \
-config=/ups/app/etcd/ssl/ca-config.json \
-profile=server /ups/app/etcd/ssl/etcd-csr.json | cfssljson -bare etcd
# 或
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server etcd-csr.json | cfssljson -bare etcd
etcd 有關證書證書列表如下
[root@pg1 ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=etcd etcd-csr.json | cfssljson -bare etcd
2020/07/29 20:02:11 [INFO] generate received request
2020/07/29 20:02:11 [INFO] received CSR
2020/07/29 20:02:11 [INFO] generating key: rsa-2048
2020/07/29 20:02:11 [INFO] encoded CSR
2020/07/29 20:02:11 [INFO] signed certificate with serial number 624986965983467912700213173636453978413637921699
2020/07/29 20:02:11 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@pg1 ssl]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem etcd.csr etcd-csr.json etcd-key.pem etcd.pem
[root@pg1 ssl]#
證書分發到其它所有節點
# 給證書讀許可權
chmod 644 /ups/app/etcd/ssl/*
cd /ups/app/etcd/ssl/
for IP in pg2 pg3;do
scp ca-key.pem ca.pem etcd.pem etcd-key.pem $IP:/ups/app/etcd/ssl/
done
Etcd安裝
三臺主機上下載並安裝ETCD，如下:
# wget -c https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz
tar -xf etcd-v3.3.22-linux-amd64.tar.gz -C /ups/app/
cd /ups/app
mv etcd-v3.3.22-linux-amd64/ etcd
chown -R root:root etcd
mkdir -p ./{bin,cfg,ssl,log,data} && mv etcd etcdctl ./bin/
# 可選
useradd etcd -M -d /ups/app/etcd -c "Etcd user" -r -s /sbin/nologin
編輯配置檔案
注意是否啟動自簽證書+SSL+HTTPS組合配置，若未啟動SSL，則修改對應http協議
節點1
- 常規（不適用SSL）
cat > /ups/app/etcd/cfg/etcd.conf <<-EOF
#[Member]
#etcd例項名稱
ETCD_NAME="etcd01"
#etcd資料儲存目錄
ETCD_DATA_DIR="/ups/app/etcd/data/default.etcd"
#叢集內部通訊使用的URL
ETCD_LISTEN_PEER_URLS="http://192.168.10.190:2380"
#供外部客戶端使用的URL
ETCD_LISTEN_CLIENT_URLS="http://192.168.10.190:2379,http://127.0.0.1:2379"
#[Clustering]
#廣播給叢集內其他成員訪問的URL
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.10.190:2380"
#廣播給外部客戶端使用的URL
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.10.190:2379"
#初始叢集成員列表
ETCD_INITIAL_CLUSTER="etcd01=http://192.168.10.190:2380,etcd02=http://192.168.10.191:2380,etcd03=http://192.168.10.192:2380"
#叢集的名稱
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-pg"
#初始叢集狀態，new表示新建叢集| existing表示加入已有叢集
ETCD_INITIAL_CLUSTER_STATE="new"
# 使用3.4.X版本時配置
# ETCD_ENABLE_V2="true"
EOF
- 使用SSL證書
cat > /ups/app/etcd/cfg/etcd.conf <<-EOF
#[Member]
#etcd例項名稱,可以隨意設定不重複值
ETCD_NAME="etcd01"
#etcd資料儲存目錄
ETCD_DATA_DIR="/ups/app/etcd/data/default.etcd"
#叢集內部通訊使用的URL
ETCD_LISTEN_PEER_URLS="https://192.168.10.190:2380"
#供外部客戶端使用的URL
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.190:2379,http://127.0.0.1:2379"
#[Clustering]
#廣播給叢集內其他成員訪問的URL
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.190:2380"
#廣播給外部客戶端使用的URL
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.190:2379"
#初始叢集成員列表
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.10.190:2380,etcd02=https://192.168.10.191:2380,etcd03=https://192.168.10.192:2380"
#叢集的名稱
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-pg"
#初始叢集狀態，new表示新建叢集| existing表示加入已有叢集
ETCD_INITIAL_CLUSTER_STATE="new"
# 使用3.4.X版本時配置
# ETCD_ENABLE_V2="true"
#[Security]
ETCD_CERT_FILE="/ups/app/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/ups/app/etcd/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/ups/app/etcd/ssl/ca.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_PEER_CERT_FILE="/ups/app/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/ups/app/etcd/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/ups/app/etcd/ssl/ca.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_AUTO_TLS="true"
ETCD_AUTO_TLS="true"
EOF
配置說明：
-
ETCD_NAME:
- etcd 叢集中的節點名，這裡可以隨意，可區分且不重複就行。
-
ETCD_LISTEN_PEER_URLS:
- 監聽的用於節點之間通訊的 URL，可監聽多個，叢集內部將通過這些 URL 進行資料互動（如選舉、資料同步等）。
-
ETCD_LISTEN_CLIENT_URLS:
- 監聽的用於客戶端通訊的 URL，同樣可以監聽多個。
-
ETCD_ADVERTISE_CLIENT_URLS:
- 建議使用的客戶端通訊 URL，該值用於 etcd 代理或 etcd 成員與 etcd 節點通訊。
-
ETCD_INITIAL_ADVERTISE_PEER_URLS:
- 建議用於節點之間通訊的 URL，節點間將以該值進行通訊。
-
ETCD_INITIAL_CLUSTER:
- 也就是叢集中所有的 initial--advertise-peer-urls 的合集。
-
ETCD_INITIAL_CLUSTER_STATE:
- 新建叢集的標誌。
-
ETCD_INITIAL_CLUSTER_TOKEN:
- 節點的 token 值，設定該值後集群將生成唯一 ID，併為每個節點也生成唯一 ID，當使用相同配置檔案再啟動一個叢集時，只要該 token 值不一樣，etcd 叢集就不會相互影響。
節點2
cat > /ups/app/etcd/cfg/etcd.conf <<-EOF
#[Member]
#etcd例項名稱
ETCD_NAME="etcd02"
#etcd資料儲存目錄
ETCD_DATA_DIR="/ups/app/etcd/data/default.etcd"
#叢集內部通訊使用的URL
ETCD_LISTEN_PEER_URLS="http://192.168.10.191:2380"
#供外部客戶端使用的URL
ETCD_LISTEN_CLIENT_URLS="http://192.168.10.191:2379,http://127.0.0.1:2379"
#[Clustering]
#廣播給叢集內其他成員訪問的URL
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.10.191:2380"
#廣播給外部客戶端使用的URL
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.10.191:2379"
#初始叢集成員列表
ETCD_INITIAL_CLUSTER="etcd01=http://192.168.10.190:2380,etcd02=http://192.168.10.191:2380,etcd03=http://192.168.10.192:2380"
#叢集的名稱
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-pg"
#初始叢集狀態，new表示新建叢集| existing表示加入已有叢集
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
配置etcd啟動檔案
etcd-V3.3.X版本
- 不適用證書
cat > /usr/lib/systemd/system/etcd.service <<-EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
#User=etcd
WorkingDirectory=/ups/app/etcd/
EnvironmentFile=-/ups/app/etcd/cfg/etcd.conf
ExecStart=/ups/app/etcd/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS} \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=\${ETCD_INITIAL_CLUSTER_STATE} \
--log-output=stderr
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
- 使用證書
cat > /usr/lib/systemd/system/etcd.service <<-EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/ups/app/etcd/
EnvironmentFile=-/ups/app/etcd/cfg/etcd.conf
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /ups/app/etcd/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS} \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=\${ETCD_INITIAL_CLUSTER_STATE} \
--auto-tls=\${ETCD_AUTO_TLS} \
--cert-file=\${ETCD_CERT_FILE} \
--key-file=\${ETCD_KEY_FILE} \
--peer-cert-file=\${ETCD_PEER_CERT_FILE} \
--peer-key-file=\${ETCD_PEER_KEY_FILE} \
--trusted-ca-file=\${ETCD_TRUSTED_CA_FILE} \
--client-cert-auth=\${ETCD_CLIENT_CERT_AUTH} \
--peer-client-cert-auth=\${ETCD_PEER_CLIENT_CERT_AUTH} \
--peer-trusted-ca-file=\${ETCD_PEER_TRUSTED_CA_FILE} \
--peer-auto-tls=\${ETCD_PEER_AUTO_TLS} "
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
etcd-V3.4.X版本
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
# User=etcd
WorkingDirectory=/ups/app/etcd/
EnvironmentFile=-/ups/app/etcd/cfg/etcd.conf
ExecStart=/ups/app/etcd/bin/etcd --log-output=stderr
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
在3.4.X的版本中，不能如下圖同時配置引數檔案和啟動帶選項
啟動
服務方式啟動etcd
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd
systemctl daemon-reload && systemctl restart etcd && systemctl status etcd
指令碼方式啟動etcd
pg1主機建立 main.sh 啟動指令碼，如下:
etcd --name etcd01 \
--initial-advertise-peer-urls http://192.168.10.190:2380 \
--listen-peer-urls http://192.168.10.190:2380 \
--listen-client-urls http://192.168.10.190:2379,http://127.0.0.1:2379 \
--advertise-client-urls http://192.168.10.190:2379 \
--initial-cluster-token etcd-cluster-pg \
--initial-cluster etcd01=http://192.168.10.190:2380,etcd02=http://192.168.10.191:2380,etcd03=http://192.168.10.192:2380 \
--initial-cluster-state new \
--enable-v2
pg2 主機建立 main.sh啟動指令碼，如下:
cd /ups/app/etcd
./etcd --name etcd02 \
--initial-advertise-peer-urls http://192.168.10.191:2380 \
--listen-peer-urls http://192.168.10.191:2380 \
--listen-client-urls http://192.168.10.191:2379,http://127.0.0.1:2379 \
--advertise-client-urls http://192.168.10.191:2379 \
--initial-cluster-token etcd-cluster-pg \
--initial-cluster etcd01=http://192.168.10.190:2380,etcd02=http://192.168.10.191:2380,etcd03=http://192.168.10.192:2380 \
--initial-cluster-state existing \
--enable-v2
pg3 主機建立 main.sh 啟動指令碼，如下:
cd /ups/app/etcd
./etcd --name etcd03 \
--initial-advertise-peer-urls http://192.168.10.192:2380 \
--listen-peer-urls http://192.168.10.192:2380 \
--listen-client-urls http://192.168.10.192:2379,http://127.0.0.1:2379 \
--advertise-client-urls http://192.168.10.192:2379 \
--initial-cluster-token etcd-cluster-pg \
--initial-cluster etcd01=http://192.168.10.190:2380,etcd02=http://192.168.10.191:2380,etcd03=http://192.168.10.192:2380 \
--initial-cluster-state new \
--enable-v2
使用證書
etcd --name etcd01 \
--initial-advertise-peer-urls https://192.168.10.190:2380 \
--listen-peer-urls https://192.168.10.190:2380 \
--listen-client-urls https://192.168.10.190:2379,http://127.0.0.1:2379 \
--advertise-client-urls https://192.168.10.190:2379 \
--initial-cluster-token etcd-cluster-pg \
--initial-cluster etcd01=https://192.168.10.190:2380,etcd02=https://192.168.10.191:2380,etcd03=https://192.168.10.192:2380 \
--initial-cluster-state new \
--cert-file=/ups/app/etcd/ssl/etcd.pem \
--key-file=/ups/app/etcd/ssl/etcd-key.pem \
--peer-cert-file=/ups/app/etcd/ssl/etcd.pem \
--peer-key-file=/ups/app/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/ups/app/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/ups/app/etcd/ssl/ca.pem
etcd --name etcd02 \
--initial-advertise-peer-urls https://192.168.10.191:2380 \
--listen-peer-urls https://192.168.10.191:2380 \
--listen-client-urls https://192.168.10.191:2379,http://127.0.0.1:2379 \
--advertise-client-urls https://192.168.10.191:2379 \
--initial-cluster-token etcd-cluster-pg \
--initial-cluster etcd01=https://192.168.10.190:2380,etcd02=https://192.168.10.191:2380,etcd03=https://192.168.10.192:2380 \
--initial-cluster-state new \
--cert-file=/ups/app/etcd/ssl/etcd.pem \
--key-file=/ups/app/etcd/ssl/etcd-key.pem \
--peer-cert-file=/ups/app/etcd/ssl/etcd.pem \
--peer-key-file=/ups/app/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/ups/app/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/ups/app/etcd/ssl/ca.pem
etcd --name etcd03 \
--initial-advertise-peer-urls https://192.168.10.192:2380 \
--listen-peer-urls https://192.168.10.192:2380 \
--listen-client-urls https://192.168.10.192:2379,http://127.0.0.1:2379 \
--advertise-client-urls https://192.168.10.192:2379 \
--initial-cluster-token etcd-cluster-pg \
--initial-cluster etcd01=https://192.168.10.190:2380,etcd02=https://192.168.10.191:2380,etcd03=https://192.168.10.192:2380 \
--initial-cluster-state new \
--cert-file=/ups/app/etcd/ssl/etcd.pem \
--key-file=/ups/app/etcd/ssl/etcd-key.pem \
--peer-cert-file=/ups/app/etcd/ssl/etcd.pem \
--peer-key-file=/ups/app/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/ups/app/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/ups/app/etcd/ssl/ca.pem
啟動etcd，如下:
# cat start.sh
#!/bin/sh
cd /ups/app/etcd
sh ./main.sh > etcd.log 2>&1 &
# 執行啟動指令碼
sh start.sh
驗證
# 檢視成員
etcdctl \
--ca-file=/ups/app/etcd/ssl/ca.pem \
--cert-file=/ups/app/etcd/ssl/etcd.pem \
--key-file=/ups/app/etcd/ssl/etcd-key.pem \
--endpoints=https://192.168.10.190:2379 member list
# 檢視叢集狀態
etcdctl \
--endpoints=https://192.168.10.190:2379 \
--cert-file=/ups/app/etcd/ssl/etcd.pem \
--ca-file=/ups/app/etcd/ssl/ca.pem \
--key-file=/ups/app/etcd/ssl/etcd-key.pem \
cluster-health
ETCDCTL_API=3 etcdctl --endpoints=http://192.168.10.190:2379,http://192.168.10.191:2379,http://192.168.10.192:2379 endpoint health
ETCDCTL_API=2 etcdctl --endpoints "http://192.168.10.190:2379,http://192.168.10.191:2379,http://192.168.10.192:2379" member list
# -- etcd3.4
/ups/app/etcd/bin/etcd --version
/ups/app/etcd/bin/etcdctl endpoint health
/ups/app/etcd/bin/etcdctl endpoint status
/ups/app/etcd/bin/etcdctl member list
etcdctl管理工具
各個版本不同命令選項
語法
AME:
etcdctl - A simple command line client for etcd.
WARNING:
Environment variable ETCDCTL_API is not set; defaults to etcdctl v2.
Set environment variable ETCDCTL_API=3 to use v3 API or ETCDCTL_API=2 to use v2 API.
USAGE:
etcdctl [global options] command [command options] [arguments...]
VERSION:
3.3.22
COMMANDS:
backup backup an etcd directory
cluster-health check the health of the etcd cluster
mk make a new key with a given value
mkdir make a new directory
rm remove a key or a directory
rmdir removes the key if it is an empty directory or a key-value pair
get retrieve the value of a key
ls retrieve a directory
set set the value of a key
setdir create a new directory or update an existing directory TTL
update update an existing key with a given value
updatedir update an existing directory
watch watch a key for changes
exec-watch watch a key for changes and exec an executable
member member add, remove and list subcommands
user user add, grant and revoke subcommands
role role add, grant and revoke subcommands
auth overall auth controls
help, h Shows a list of commands or help for one command
GLOBAL OPTIONS:
--debug output cURL commands which can be used to reproduce the request
--no-sync don't synchronize cluster information before sending request
--output simple, -o simple output response in the given format (simple, `extended` or `json`) (default: "simple")
--discovery-srv value, -D value domain name to query for SRV records describing cluster endpoints
--insecure-discovery accept insecure SRV records describing cluster endpoints
--peers value, -C value DEPRECATED - "--endpoints" should be used instead
--endpoint value DEPRECATED - "--endpoints" should be used instead
--endpoints value a comma-delimited list of machine addresses in the cluster (default: "http://127.0.0.1:2379,http://127.0.0.1:4001")
--cert-file value identify HTTPS client using this SSL certificate file
--key-file value identify HTTPS client using this SSL key file
--ca-file value verify certificates of HTTPS-enabled servers using this CA bundle
--username value, -u value provide username[:password] and prompt if password is not supplied.
--timeout value connection timeout per request (default: 2s)
--total-timeout value timeout for the command execution (except watch) (default: 5s)
--help, -h show help
--version, -v print the version
[root@pg3 ~]# etcdctl ls -h
NAME:
etcdctl ls - retrieve a directory
USAGE:
etcdctl ls [command options] [key]
OPTIONS:
--sort returns result in sorted order
--recursive, -r returns all key names recursively for the given path
-p append slash (/) to directories
--quorum, -q require quorum for get request
[root@pg3 ~]#
export ETCDCTL_API=3
[root@pg1 ~]# etcdctl --help
NAME:
etcdctl - A simple command line client for etcd3.
USAGE:
etcdctl
VERSION:
3.3.22
API VERSION:
3.3
COMMANDS:
get Gets the key or a range of keys
put Puts the given key into the store
del Removes the specified key or range of keys [key, range_end)
txn Txn processes all the requests in one transaction
compaction Compacts the event history in etcd
alarm disarm Disarms all alarms
alarm list Lists all alarms
defrag Defragments the storage of the etcd members with given endpoints
endpoint health Checks the healthiness of endpoints specified in `--endpoints` flag
endpoint status Prints out the status of endpoints specified in `--endpoints` flag
endpoint hashkv Prints the KV history hash for each endpoint in --endpoints
move-leader Transfers leadership to another etcd cluster member.
watch Watches events stream on keys or prefixes
version Prints the version of etcdctl
lease grant Creates leases
lease revoke Revokes leases
lease timetolive Get lease information
lease list List all active leases
lease keep-alive Keeps leases alive (renew)
member add Adds a member into the cluster
member remove Removes a member from the cluster
member update Updates a member in the cluster
member list Lists all members in the cluster
snapshot save Stores an etcd node backend snapshot to a given file
snapshot restore Restores an etcd member snapshot to an etcd directory
snapshot status Gets backend snapshot status of a given file
make-mirror Makes a mirror at the destination etcd cluster
migrate Migrates keys in a v2 store to a mvcc store
lock Acquires a named lock
elect Observes and participates in leader election
auth enable Enables authentication
auth disable Disables authentication
user add Adds a new user
user delete Deletes a user
user get Gets detailed information of a user
user list Lists all users
user passwd Changes password of user
user grant-role Grants a role to a user
user revoke-role Revokes a role from a user
role add Adds a new role
role delete Deletes a role
role get Gets detailed information of a role
role list Lists all roles
role grant-permission Grants a key to a role
role revoke-permission Revokes a key from a role
check perf Check the performance of the etcd cluster
help Help about any command
OPTIONS:
--cacert="" verify certificates of TLS-enabled secure servers using this CA bundle
--cert="" identify secure client using this TLS certificate file
--command-timeout=5s timeout for short running command (excluding dial timeout)
--debug[=false] enable client-side debug logging
--dial-timeout=2s dial timeout for client connections
-d, --discovery-srv="" domain name to query for SRV records describing cluster endpoints
--endpoints=[127.0.0.1:2379] gRPC endpoints
--hex[=false] print byte strings as hex encoded strings
--insecure-discovery[=true] accept insecure SRV records describing cluster endpoints
--insecure-skip-tls-verify[=false] skip server certificate verification
--insecure-transport[=true] disable transport security for client connections
--keepalive-time=2s keepalive time for client connections
--keepalive-timeout=6s keepalive timeout for client connections
--key="" identify secure client using this TLS key file
--user="" username[:password] for authentication (prompt if password is not supplied)
-w, --write-out="simple" set the output format (fields, json, protobuf, simple, table)
[root@pg1 ~]#
示例
# 檢視所有資料
etcdctl ls -recursive
etcdctl --endpoints=http://127.0.0.1:2379 ls -recursive
# 刪除 key名稱 /service
root@pg1 ~]# etcdctl ls -recursive
/service
/service/etcd-cluster-pg
/service/etcd-cluster-pg/config
/service/etcd-cluster-pg/optime
/service/etcd-cluster-pg/optime/leader
/service/etcd-cluster-pg/history
/service/etcd-cluster-pg/members
/service/etcd-cluster-pg/initialize
[root@pg1 ~]# etcdctl rm /service
Error: 102: Not a file (/service) [6680]
[root@pg1 ~]# etcdctl rm -recursive /service
[root@pg1 ~]# etcdctl ls -recursive
[root@pg1 ~]#
Patroni 安裝
export LD_LIBRARY_PATH=/ups/app/python3/lib:${LD_LIBRARY_PATH}
cd /ups/app/python3/bin
./python3 -m pip install --upgrade setuptools
./python3 -m pip install --upgrade pip
./python3 -m pip install psycopg2_binary
./python3 -m pip install patroni[etcd,consul]
# -- 或者
/ups/app/python3/bin/pip3 install psycopg2-binary -i https://mirrors.aliyun.com/pypi/simple/
/ups/app/python3/bin/pip3 install patroni -i https://mirrors.aliyun.com/pypi/simple/
# OR
/ups/app/python3/bin/pip3 install patroni[etcd,consul] -i https://mirrors.aliyun.com/pypi/simple/
過程
[root@pg1 bin]# cd /ups/app/python3/bin
[root@pg1 bin]# ./pip3 install patroni[etcd] -i https://mirrors.aliyun.com/pypi/simple/
Looking in indexes: https://mirrors.aliyun.com/pypi/simple/
Collecting patroni[etcd]
Downloading https://mirrors.aliyun.com/pypi/packages/25/01/e4656c541ac648a530fc1b6094324969f9f2ed8d7005ad0fa2598cbf1199/patroni-1.6.5-py3-none-any.whl (178kB)
|████████████████████████████████| 184kB 425kB/s
Collecting psutil>=2.0.0 (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/aa/3e/d18f2c04cf2b528e18515999b0c8e698c136db78f62df34eee89cee205f1/psutil-5.7.2.tar.gz (460kB)
|████████████████████████████████| 460kB 1.4MB/s
Collecting six>=1.7 (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl
Collecting prettytable>=0.7 (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/ef/30/4b0746848746ed5941f052479e7c23d2b56d174b82f4fd34a25e389831f5/prettytable-0.7.2.tar.bz2
Collecting click>=4.1 (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl (82kB)
|████████████████████████████████| 92kB 1.3MB/s
Collecting urllib3!=1.21,>=1.19.1 (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/9f/f0/a391d1463ebb1b233795cabfc0ef38d3db4442339de68f847026199e69d7/urllib3-1.25.10-py2.py3-none-any.whl (127kB)
|████████████████████████████████| 133kB 323kB/s
Collecting PyYAML (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz (269kB)
|████████████████████████████████| 276kB 383kB/s
Collecting cdiff (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/69/6c/301876940e760a8b46c1caacf08c298f511f517c70eec32e43f38e9cc6f5/cdiff-1.0.tar.gz
Collecting python-dateutil (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/d4/70/d60450c3dd48ef87586924207ae8907090de0b306af2bce5d134d78615cb/python_dateutil-2.8.1-py2.py3-none-any.whl (227kB)
|████████████████████████████████| 235kB 943kB/s
Collecting python-etcd<0.5,>=0.4.3; extra == "etcd" (from patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/a1/da/616a4d073642da5dd432e5289b7c1cb0963cc5dde23d1ecb8d726821ab41/python-etcd-0.4.5.tar.gz
Collecting dnspython>=1.13.0 (from python-etcd<0.5,>=0.4.3; extra == "etcd"->patroni[etcd])
Downloading https://mirrors.aliyun.com/pypi/packages/90/49/cb426577c28ca3e35332815b795a99e467523843fc83cc85ca0d6be2515a/dnspython-2.0.0-py3-none-any.whl (208kB)
|████████████████████████████████| 215kB 2.8MB/s
Installing collected packages: psutil, six, prettytable, click, urllib3, PyYAML, cdiff, python-dateutil, dnspython, python-etcd, patroni
Running setup.py install for psutil ... done
Running setup.py install for prettytable ... done
Running setup.py install for PyYAML ... done
Running setup.py install for cdiff ... done
Running setup.py install for python-etcd ... done
Successfully installed PyYAML-5.3.1 cdiff-1.0 click-7.1.2 dnspython-2.0.0 patroni-1.6.5 prettytable-0.7.2 psutil-5.7.2 python-dateutil-2.8.1 python-etcd-0.4.5 six-1.15.0 urllib3-1.25.10
WARNING: You are using pip version 19.2.3, however version 20.1.1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
結果確認
[root@temp bin]# ./pip3 install patroni -i https://mirrors.aliyun.com/pypi/simple/
Looking in indexes: https://mirrors.aliyun.com/pypi/simple/
Requirement already satisfied: patroni in /ups/app/python3/lib/python3.8/site-packages (1.6.5)
Requirement already satisfied: PyYAML in /ups/app/python3/lib/python3.8/site-packages (from patroni) (5.3.1)
Requirement already satisfied: click>=4.1 in /ups/app/python3/lib/python3.8/site-packages (from patroni) (7.1.2)
Requirement already satisfied: prettytable>=0.7 in /ups/app/python3/lib/python3.8/site-packages (from patroni) (0.7.2)
Requirement already satisfied: cdiff in /ups/app/python3/lib/python3.8/site-packages (from patroni) (1.0)
Requirement already satisfied: python-dateutil in /ups/app/python3/lib/python3.8/site-packages (from patroni) (2.8.1)
Requirement already satisfied: psutil>=2.0.0 in /ups/app/python3/lib/python3.8/site-packages (from patroni) (5.7.2)
Requirement already satisfied: urllib3!=1.21,>=1.19.1 in /ups/app/python3/lib/python3.8/site-packages (from patroni) (1.25.10)
Requirement already satisfied: six>=1.7 in /ups/app/python3/lib/python3.8/site-packages (from patroni) (1.15.0)
[root@temp bin]#
find / -name 'patroni'
/ups/app/python3/bin/patroni
/ups/app/python3/lib/python3.8/site-packages/patroni
服務檔案配置
vi /usr/lib/systemd/system/patroni.service
cat <<-EOF >/usr/lib/systemd/system/patroni.service
[Unit]
Description=Runners to orchestrate a high-availability PostgreSQL - patroni
Documentation=https://patroni.readthedocs.io/en/latest/index.html
After=syslog.target network.target etcd.target
Wants=network-online.target
[Service]
Type=simple
User=postgres
Group=postgres
ExecStart=/ups/app/python3/bin/patroni /etc/patroni/patroni.yml
ExecReload=/bin/kill -s HUP $MAINPID
KillMode=process
KillSignal=SIGINT
LimitNOFILE=65536
Restart=on-abnormal
RestartSec=30s
TimeoutSec=0
[Install]
WantedBy=multi-user.target
EOF
patroni讀取PG引數順序
- postgresql.base.conf
- postgresql.conf
- postgresql.auto.conf
- run-time parameter (即執行時alter命令設定的屬性)
配置patroni檔案
Patroni 使用的是YAML的方式來進行配置，配置檔案的嚴謹
mkdir -p /etc/patroni
chown -R postgres:postgres /etc/patroni
vi /etc/patroni/patroni.yml
grep -Ev "^[ \t]*(#|$)" /etc/patroni/patroni.yml
啟動patroni服務時，將$PGDATA/postgresql.conf重新命名為$PGDATA/postgresql.base.conf，然後將/etc/patroni.yml檔案中配置的postgresql項內容覆蓋寫入到$PGDATA/postgresql.conf檔案
配置說明
配置檔案的專案很多，這裡值關心 global 和 etcd的配置
- Global 設定
name 叢集名叢集內的機器必須唯一，每臺機器有自己的名字
namespace 儲存配置資訊的區域路徑（請保持預設）
scope 叢集的名字
- log 的配置
level 設定日誌的等級
format 設定日誌的等級 預設的設定是 asctime levelname message
dateformat 設定時間格式
dir 要寫入程式日誌的目錄，目錄必須存在並且是patroni 使用者編寫並且可以由您設定此值。應用程式將預設保留4個25MB 的日誌。
file_num 要保留的日誌的數量
file_size patroni.log的尺寸
loggers: 定義允許日誌等級
- 引導配置：
DCS: 在叢集的全域性配置，更改引數需要在 DCS 中或聽過API 進行更改。
loop_wait 迴圈休眠的描述 預設 10秒
ttl: TTL獲取先導鎖。可以將其視為啟動自動故障轉移過程之前的時間長度。預設值:30
retry_timeout: 分散式程式和POSTGRESQL 之間的失聯後多長時間不觸發切換。
maximum_lag_on_failover:從庫和主庫之間在可以能進行主從切換中執行的位元組差距。
master_start_timeout 主庫在故障轉移中的時間容忍度，loop_wait + master_start_timeout+loop_wait
synchronous_mode 開啟這個模式將選擇與主庫最接近的從庫作為可的新主庫
synchronous_mode_strict ：開啟這個模式將如果發現沒有和主庫進行資料複製的從庫，則主庫將禁止寫入資料。
1主機配置yml檔案
scope: etcd-cluster-pg
namespace: /service/
name: pg1
restapi:
listen: 192.168.10.190:8008
connect_address: 192.168.10.190:8008
etcd:
#Provide host to do the initial discovery of the cluster topology:
# host: 192.168.10.190:2379
hosts: 192.168.10.190:2379,192.168.10.191:2379,192.168.10.192:2379
# protocol: https
# cacert: /ups/app/etcd/ssl/ca.pem
# cert: /ups/app/etcd/ssl/etcd.pem
# key: /ups/app/etcd/ssl/etcd-key.pem
bootstrap:
# this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
# and all other cluster members will use it as a `global configuration`
dcs:
ttl: 30
loop_wait: 10
retry_timeout: 10
maximum_lag_on_failover: 1048576
master_start_timeout: 300
synchronous_mode: false
postgresql:
use_pg_rewind: true
use_slots: true
parameters:
port: 2020
listen_addresses: "*"
wal_level: locical
hot_standby: "on"
wal_keep_segments: 64
max_wal_senders: 10
max_replication_slots: 10
wal_log_hints: "on"
# archive_mode: "on"
hot_standby: on
# archive_timeout: 1800s
postgresql:
listen: 0.0.0.0:2020
connect_address: 192.168.10.190:2020
data_dir: /ups/data/pgdata/12/pg_root
bin_dir: /ups/app/postgresql/pgsql-12/bin
# config_dir: /ups/data/pgdata/12/pg_root
pgpass: /home/postgres/.pgpass
authentication:
replication:
username: sync
password: sync12345
superuser:
username: postgres
password: postgres
#rewind: # Has no effect on postgres 10 and lower
#username: pg_rewind
#password:
tags:
nofailover: false
noloadbalance: false
clonefrom: false
nosync: false
2主機配置yml檔案
scope: etcd-cluster-pg
namespace: /service/
name: pg2
restapi:
listen: 192.168.10.191:8008
connect_address: 192.168.10.191:8008
etcd:
#Provide host to do the initial discovery of the cluster topology:
hosts: 192.168.10.190:2379,192.168.10.191:2379,192.168.10.192:2379
# protocol: https
# cacert: /ups/app/etcd/ssl/ca.pem
# cert: /ups/app/etcd/ssl/etcd.pem
# key: /ups/app/etcd/ssl/etcd-key.pem
bootstrap:
# this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
# and all other cluster members will use it as a `global configuration`
dcs:
ttl: 30
loop_wait: 10
retry_timeout: 10
maximum_lag_on_failover: 1048576
master_start_timeout: 300
synchronous_mode: false
postgresql:
use_pg_rewind: true
use_slots: true
parameters:
wal_level: locical
hot_standby: "on"
wal_keep_segments: 64
max_wal_senders: 10
max_replication_slots: 10
wal_log_hints: "on"
archive_mode: "on"
hot_standby: on
archive_timeout: 1800s
postgresql:
listen: 0.0.0.0:2020
connect_address: 192.168.10.191:2020
data_dir: /ups/data/pgdata/12/pg_root
bin_dir: /ups/app/postgresql/pgsql-12/bin
#config_dir: /ups/data/pgdata/12/pg_root
pgpass: /home/postgres/.pgpass
authentication:
replication:
username: sync
password: sync12345
superuser:
username: postgres
password:
rewind: # Has no effect on postgres 10 and lower
username: pg_rewind
password:
tags:
nofailover: false
noloadbalance: false
clonefrom: false
nosync: false
包括全域性引數、restapi模組引數、etcd模組引數、bootstrap啟動引數、postgresql模組引數，主要引數解釋如下:
- scope: 標記cluster名稱，同 postgresql.conf 的 cluster_name 引數，二級目錄名:
Etcd:/<namespace>/<scope>/config - namespace: 一級目錄名:
Etcd:/<namespace>/<scope>/config - name: patroni節點名稱
DCS: Dynamic Configuration settings
更多引數解釋詳見: YAML Configuration Settings。
其它節點需修改全域性引數name、restapi模組的listen和connect_address引數、etcd模組的host引數，以及postgresql模組的connect_address引數。
啟動服務
三臺主機分別啟動 patroni ，如下:
/ups/app/python3/bin/patroni /etc/patroni.yml> /tmp/pg_patroni.log 2>&1 &
/ups/app/python3/bin/patroni /etc/patroni.yml> /tmp/pg_patroni.log 2>&1 &
/ups/app/python3/bin/patroni /etc/patroni.yml> /tmp/pg_patroni.log 2>&1 &
# 服務方式啟動
systemctl daemon-reload && systemctl start patroni
啟動Patroni，在host1上
- Patroni1 把本地PostgreSQL(postgresql1)的資訊寫入etcd.
- Patroni1 監測到資料庫目錄(/home/rudi/pgdata/)是空的，於是初始化資料庫(initdb -D /home/rudi/pgdata)
- Patroni1 配置本地資料庫相關的配置檔案，例如：postgresql.conf, pg_hba.conf
- Patroni1 啟動本地資料庫(postgresql1): pg_ctl -D /home/rudi/pgdata start
- Patroni1 把本地資料庫(postgresql1)設定為主資料庫(Primary)
啟動Patroni，在host2/host3上
- Patroni2/Patroni3 基於postgresql1做資料庫備份(pg_basebackup)，建立各自的本地資料庫
- Patroni2/Patroni3 配置本地資料庫相關的配置檔案，例如：postgresql.conf, pg_hba.conf
- Patroni2 啟動postgresql2，作為從庫(Standby)
- Patroni3 啟動postgresql3，作為從庫(Standby)
建議手工方式配置流複製，不建議通過patroni方式配置主從環境
Patronictl 基本操作
語法
[root@pg3 ~]# patronictl --help
Usage: patronictl [OPTIONS] COMMAND [ARGS]...
Options:
-c, --config-file TEXT Configuration file
-d, --dcs TEXT Use this DCS
-k, --insecure Allow connections to SSL sites without certs
--help Show this message and exit.
Commands:
configure Create configuration file
dsn Generate a dsn for the provided member, defaults to a dsn of...
edit-config Edit cluster configuration
failover Failover to a replica
flush Discard scheduled events (restarts only currently)
history Show the history of failovers/switchovers
list List the Patroni members for a given Patroni
pause Disable auto failover
query Query a Patroni PostgreSQL member
reinit Reinitialize cluster member
reload Reload cluster member configuration
remove Remove cluster from DCS
restart Restart cluster member
resume Resume auto failover
scaffold Create a structure for the cluster in DCS
show-config Show cluster configuration
switchover Switchover to a replica
version Output version of patronictl command or a running Patroni...
[root@pg3 ~]#
示例
patronictl -c /etc/patroni/patroni_postgresql.yml show-config
patronictl -c /etc/patroni/patroni_postgresql.yml list
patronictl -c /etc/patroni/patroni_postgresql.yml edit-config
# 刪除屬性
patronictl -c /etc/patroni/patroni_postgresql.yml edit-config -s postgresql.parameters.synchronous_standby_names=null
# 重啟資料庫
patronictl -c /etc/patroni/patroni_postgresql.yml restart pgha
檢視 patroni 叢集
patronictl -c /etc/patroni.yml list
patronictl -c /etc/patroni.yml list etcd-cluster-pg
patronictl -d etcd://pg1:2379 list etcd-cluster-pg
# 輸出結果
+ Cluster: etcd-cluster-pg (6854432365693308402) -+----+-----------+
| Member | Host | Role | State | TL | Lag in MB |
+--------+---------------------+--------+---------+----+-----------+
| pg1 | 192.168.10.190:2020 | Leader | running | 1 | |
| pg2 | 192.168.10.191:2020 | | running | 1 | 0 |
| pg3 | 192.168.10.192:2020 | | running | 1 | 0 |
+--------+---------------------+--------+---------+----+-----------+
[root@pg1 ~]# etcdctl ls --recursive --sort -p /service
/service/etcd-cluster-pg/
/service/etcd-cluster-pg/config
/service/etcd-cluster-pg/initialize
/service/etcd-cluster-pg/leader
/service/etcd-cluster-pg/members/
/service/etcd-cluster-pg/members/pg1
/service/etcd-cluster-pg/members/pg2
/service/etcd-cluster-pg/members/pg3
/service/etcd-cluster-pg/optime/
/service/etcd-cluster-pg/optime/leader
[root@pg1 ~]#
[root@pg1 ~]# etcdctl get /service/etcd-cluster-pg/members/pg1
{"conn_url":"postgres://192.168.10.190:2020/postgres","api_url":"http://192.168.10.190:8008/patroni","state":"running","role":"master","version":"1.6.5","xlog_location":201335568,"timeline":1}
檢視 cluster 狀態
curl -s "http://192.168.10.190:8008/cluster" | jq .
curl -s "http://192.168.10.191:8008/cluster" | jq .
curl -s "http://192.168.10.192:8008/cluster" | jq .
[postgres@pg1 ~]$ curl -s "http://192.168.10.190:8008/patroni" | jq .
{
"members": [
{
"name": "pgsql12_pg1",
"role": "leader",
"state": "running",
"api_url": "http://192.168.10.190:8008/patroni",
"host": "192.168.10.190",
"port": 2020,
"timeline": 2
},
{
"name": "pgsql12_pg2",
"role": "replica",
"state": "running",
"api_url": "http://192.168.10.191:8008/patroni",
"host": "192.168.10.191",
"port": 2020,
"timeline": 2,
"lag": 0
},
{
"name": "pgsql12_pg3",
"role": "replica",
"state": "running",
"api_url": "http://192.168.10.192:8008/patroni",
"host": "192.168.10.192",
"port": 2020,
"timeline": 2,
"lag": 0
}
]
}
檢視 patroni 節點狀態
curl -s "http://192.168.10.190:8008/patroni" | jq .
curl -s "http://192.168.10.191:8008/patroni" | jq .
curl -s "http://192.168.10.192:8008/patroni" | jq .
[root@pg1 ~]# curl -s "http://192.168.10.190:8008/patroni" | jq .
{
"state": "running",
"postmaster_start_time": "2020-07-29 10:25:32.214 CST",
"role": "master",
"server_version": 120002,
"cluster_unlocked": false,
"xlog": {
"location": 201335568
},
"timeline": 1,
"replication": [
{
"usename": "sync",
"application_name": "pg2",
"client_addr": "192.168.10.191",
"state": "streaming",
"sync_state": "sync",
"sync_priority": 1
},
{
"usename": "sync",
"application_name": "pg3",
"client_addr": "192.168.10.192",
"state": "streaming",
"sync_state": "potential",
"sync_priority": 1
}
],
"database_system_identifier": "6854432365693308402",
"patroni": {
"version": "1.6.5",
"scope": "etcd-cluster-pg"
}
}
更多查詢命令參考 Patroni REST API。