配置Etcd集群和TLS認證
阿新 • • 發佈：2018-04-09
k8s etcd tls 由於後續準備在內網開發和測試環境采用二進制方式部署K8S相關組件，並考慮各組件的高可用性和安全性問題,本節介紹etcd服務的集群及tls配置。
一、安裝環境介紹
二、Etcd二進制軟件包下載地址:
https://github.com/coreos/etcd/releases/download/v3.3.2/etcd-v3.3.2-linux-amd64.tar.gz
三、安裝與配置etcd組件
1、刪除rpm版本的軟件包、設置各自的主機名及時間
# yum -y remove etcd # hostnamectl set-hostname vm1 # timedatectl set-timezone Asia/Shanghai # cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.115.5 vm1 192.168.115.6 vm2 192.168.115.7 vm3 # ntpdate -u pool.ntp.org
2、關閉防火墻、配置秘鑰信任
# systemctl stop firewalled
# systemctl disable firewalled
# ssh-keygen
# ssh-copy-id -i /root/.ssh/id_rsa.pub root@vm2
# ssh-copy-id -i /root/.ssh/id_rsa.pub root@vm3
# date && ssh vm2 date && ssh vm3 date 
3、將etcd軟件包上傳並解壓到/usr/local/bin目錄
# cd /usr/local/src/ # tar -zxvpf etcd-v3.3.2-linux-amd64.tar.gz # cp etcd-v3.3.2-linux-amd64/{etcd,etcdctl} /usr/local/sbin/ # chmod +x /usr/local/sbin/etcd* # scp -rp /usr/local/sbin/etcd* vm2:/usr/local/sbin/ # scp -rp /usr/local/sbin/etcd* vm3:/usr/local/sbin/
4、準備配置文件
Vm1:
# cat /etc/etcd.conf name: infra0 data-dir: /data/etcd listen-client-urls: http://192.168.115.5:2379,http://127.0.0.1:2379 advertise-client-urls: http://192.168.115.5:2379,http://127.0.0.1:2379 listen-peer-urls: http://192.168.115.5:2380 initial-advertise-peer-urls: http://192.168.115.5:2380 initial-cluster: infra0=http://192.168.115.5:2380,infra1=http://192.168.115.6:2380,infra2=http://192.168.115.7:2380 initial-cluster-token: etcd-cluster-token initial-cluster-state: new
Vm2:
# cat /etc/etcd.conf
name: infra1
data-dir: /data/etcd
listen-client-urls: http://192.168.115.6:2379,http://127.0.0.1:2379
advertise-client-urls: http://192.168.115.6:2379,http://127.0.0.1:2379
listen-peer-urls: http://192.168.115.6:2380
initial-advertise-peer-urls: http://192.168.115.6:2380
initial-cluster: infra0=http://192.168.115.5:2380,infra1=http://192.168.115.6:2380,infra2=http://192.168.115.7:2380
initial-cluster-token: etcd-cluster-token
initial-cluster-state: newVM3:
# cat /etc/etcd.conf
name: infra2
data-dir: /data/etcd
listen-client-urls: http://192.168.115.7:2379,http://127.0.0.1:2379
advertise-client-urls: http://192.168.115.7:2379,http://127.0.0.1:2379
listen-peer-urls: http://192.168.115.7:2380
initial-advertise-peer-urls: http://192.168.115.7:2380
initial-cluster: infra0=http://192.168.115.5:2380,infra1=http://192.168.115.6:2380,infra2=http://192.168.115.7:2380
initial-cluster-token: etcd-cluster-token
initial-cluster-state: new
5、啟動etcd集群並測試
# mkdir -p /data/etcd
# nohup etcd --config-file=/etc/etcd.conf &
# export ETCDCTL_API=2
# etcdctl cluster-health
# etcdctl member list
# export ETCDCTL_API=3
# etcdctl --write-out=table --endpoints=192.168.115.5:2379 member list
四、配置etcd tls
1、下載cfssl工具
# mkdir ~/bin
# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
# mv cfssl_linux-amd64 /usr/local/bin/cfssl
# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfss-certinfo
# chmod +x /usr/local/bin/cfssl* 2、生成證書
# mkdir ssl
# cd ssl
# cat bulid-key.sh
echo ‘{"CN":"CA","key":{"algo":"rsa","size":2048}}‘ | cfssl gencert -initca - | cfssljson -bare ca -
echo ‘{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","server auth","client auth"]}}}‘ > ca-config.json
export ADDRESS=192.168.115.5,192.168.115.6,192.168.115.7,vm1,vm2,vm3
export NAME=server
echo ‘{"CN":"‘$NAME‘","hosts":[""],"key":{"algo":"rsa","size":2048}}‘ | cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $NAME
export ADDRESS=
export NAME=client
echo ‘{"CN":"‘$NAME‘","hosts":[""],"key":{"algo":"rsa","size":2048}}‘ | cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $NAME
# sh bulid-key.sh
# ll
total 44
-rw-r--r-- 1 root root 732 Apr 3 05:13 build-ca.sh
-rw-r--r-- 1 root root 112 Apr 3 05:13 ca-config.json
-rw-r--r-- 1 root root 883 Apr 3 05:13 ca.csr
-rw------- 1 root root 1675 Apr 3 05:13 ca-key.pem
-rw-r--r-- 1 root root 1119 Apr 3 05:13 ca.pem
-rw-r--r-- 1 root root 928 Apr 3 05:13 client.csr
-rw------- 1 root root 1675 Apr 3 05:13 client-key.pem
-rw-r--r-- 1 root root 1180 Apr 3 05:13 client.pem
-rw-r--r-- 1 root root 928 Apr 3 05:13 server.csr
-rw------- 1 root root 1679 Apr 3 05:13 server-key.pem
-rw-r--r-- 1 root root 1220 Apr 3 05:13 server.pem4、將相關的文件復制到etc節點上
# mkdir -p /etc/ssl/etcd/
# cp ./*.pem /etc/ssl/etcd/
# scp -rp /etc/ssl/etcd/ vm2:/etc/ssl/
# scp -rp /etc/ssl/etcd/ vm3:/etc/ssl/5、配置etcd啟動加載相關證書
Vm1:
# etcd --name=infra0 --data-dir=/data/etcd --listen-client-urls=https://192.168.115.5:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.115.5:2379,https://127.0.0.1:2379 --listen-peer-urls=https://192.168.115.5:2380 --initial-advertise-peer-urls=https://192.168.115.5:2380 --initial-cluster=infra0=https://192.168.115.5:2380,infra1=https://192.168.115.6:2380,infra2=https://192.168.115.7:2380 --initial-cluster-token=etcd-cluster-token --initial-cluster-state=new --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --peer-cert-file=/etc/ssl/etcd/server.pem --peer-key-file=/etc/ssl/etcd/server-key.pem --trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-client-cert-auth=true --client-cert-auth=truevm2：
# etcd --name=infra1 --data-dir=/data/etcd --listen-client-urls=https://192.168.115.6:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.115.6:2379,https://127.0.0.1:2379 --listen-peer-urls=https://192.168.115.6:2380 --initial-advertise-peer-urls=https://192.168.115.6:2380 --initial-cluster=infra0=https://192.168.115.5:2380,infra1=https://192.168.115.6:2380,infra2=https://192.168.115.7:2380 --initial-cluster-token=etcd-cluster-token --initial-cluster-state=new --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --peer-cert-file=/etc/ssl/etcd/server.pem --peer-key-file=/etc/ssl/etcd/server-key.pem --trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-client-cert-auth=true --client-cert-auth=trueVm3:
# etcd --name=infra2 --data-dir=/data/etcd --listen-client-urls=https://192.168.115.7:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.115.7:2379,https://127.0.0.1:2379 --listen-peer-urls=https://192.168.115.7:2380 --initial-advertise-peer-urls=https://192.168.115.7:2380 --initial-cluster=infra0=https://192.168.115.5:2380,infra1=https://192.168.115.6:2380,infra2=https://192.168.115.7:2380 --initial-cluster-token=etcd-cluster-token --initial-cluster-state=new --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --peer-cert-file=/etc/ssl/etcd/server.pem --peer-key-file=/etc/ssl/etcd/server-key.pem --trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-client-cert-auth=true --client-cert-auth=true6、驗證
# export ETCDCTL_API=2
# etcdctl --cert-file=/etc/ssl/etcd/client.pem --key-file=/etc/ssl/etcd/client-key.pem --ca-file=/etc/ssl/etcd/ca.pem --endpoints=https://192.168.115.5:2379,https://192.168.115.6:2379,https://192.168.115.7:2379 cluster-health
# export ETCDCTL_API=3
# etcdctl --write-out=table --cert=/etc/ssl/etcd/client.pem --key=/etc/ssl/etcd/client-key.pem --cacert=/etc/ssl/etcd/ca.pem --endpoints=https://192.168.115.5:2379,https://192.168.115.6:2379,https://192.168.115.7:2379
member list
6、配置自啟動腳本
# cat /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/data/etcd/
EnvironmentFile=-/etc/etcd.conf
User=root
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/local/sbin/etcd --name=infra0 --data-dir=/data/etcd --listen-client-urls=https://192.168.115.5:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.115.5:2379,https://127.0.0.1:2379 --listen-peer-urls=https://192.168.115.5:2380 --initial-advertise-peer-urls=https://192.168.115.5:2380 --initial-cluster=infra0=https://192.168.115.5:2380,infra1=https://192.168.115.6:2380,infra2=https://192.168.115.7:2380 --initial-cluster-token=etcd-cluster-token --initial-cluster-state=new --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --peer-cert-file=/etc/ssl/etcd/server.pem --peer-key-file=/etc/ssl/etcd/server-key.pem --trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-client-cert-auth=true --client-cert-auth=true"
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target# systemctl daemon-reload 
參考：
https://coreos.com/os/docs/latest/generate-self-signed-certificates.html
配置Etcd集群和TLS認證