kubernetes 二進位制包安裝
阿新 • • 發佈：2019-03-19
環境
| 角色 | 主機名 | 內網 IP | 叢集 IP | 作業系統 | 服務 | 執行目錄 |
|---|---|---|---|---|---|---|
| 部署機 k8s-master | master120 | 10.0.4.120 | - | CentOS | kube-apiserver kube-scheduler kube-controller-manager | /opt/kubernetes/ |
| etcd-node | etcd121 | 10.0.4.121 | 10.10.10.121 | CentOS | etcd | /opt/etcd/ |
| etcd-node | etcd122 | 10.0.4.122 | 10.10.10.122 | CentOS | etcd | /opt/etcd/ |
| etcd-node | etcd123 | 10.0.4.123 | 10.10.10.123 | CentOS | etcd | /opt/etcd/ |
| k8s-node | node124 | 10.0.4.124 | - | CentOS | docker flannel kubelet kube-proxy | /opt/kubernetes/ |
| k8s-node | node125 | 10.0.4.125 | - | CentOS | docker flannel kubelet kube-proxy | /opt/kubernetes/ |
| k8s-node | node126 | 10.0.4.126 | - | CentOS | docker flannel kubelet kube-proxy | /opt/kubernetes/ |
- 全部伺服器關閉 firewalld 和 selinux，部署機(master120)可免密 ssh 登陸其他伺服器
- 軟體版本
- CentOS: 7.6
- etcd: 3.3.12
- docker: ce-18.09.3
- flannel: 0.11.0
- kubernetes: 1.12.6
部署 etcd 叢集
-
在部署機(master120)上操作下面步驟
-
建立 etcd 部署目錄
mkdir /home/deploy/etcd/{bin,cfg,ssl} -p -
安裝 cfssl 工具
cd /home/deploy wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64 mv cfssl_linux-amd64 /usr/local/bin/cfssl mv cfssljson_linux-amd64 /usr/local/bin/cfssljson mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo -
建立 ca-config.json 檔案
cat > /home/deploy/etcd/ssl/ca-config.json <<EOF { "signing": { "default": { "expiry": "87600h" }, "profiles": { "etcd_nodes": { "expiry": "87600h", "usages": [ "signing", "key encipherment", "server auth", "client auth" ] } } } } EOF -
建立 ca-csr.json 檔案
cat > /home/deploy/etcd/ssl/ca-csr.json <<EOF { "CN": "etcd CA", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "Beijing", "ST": "Beijing" } ] } EOF -
建立 server-csr.json 檔案
cat > /home/deploy/etcd/ssl/server-csr.json <<EOF { "CN": "etcd", "hosts": [ "10.0.4.121", "10.0.4.122", "10.0.4.123" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "BeiJing", "ST": "BeiJing" } ] } EOF -
生成證書
cd /home/deploy/ssl # 生成 ca.pem ca-key.pem cfssl gencert -initca ca-csr.json | cfssljson -bare ca - # 生成 server.pem server-key.pem cfssl gencert \ -ca=ca.pem \ -ca-key=ca-key.pem \ -config=ca-config.json \ -profile=etcd \ server-csr.json | cfssljson -bare server -
下載 etcd 二進位制包
cd /home/deploy/ wget https://github.com/etcd-io/etcd/releases/download/v3.3.12/etcd-v3.3.12-linux-amd64.tar.gz tar zxf etcd-v3.3.12-linux-amd64.tar.gz scp etcd-v3.3.12-linux-amd64/{etcd,etcdctl} etcd/bin/ -
建立 etcd 配置檔案
cat > /home/deploy/etcd/cfg/etcd <<EOF #[Member] ETCD_NAME="etcdXXX" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://10.10.10.XXX:2380" ETCD_LISTEN_CLIENT_URLS="https://10.0.4.XXX:2379" # #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.10.10.XXX:2380" ETCD_ADVERTISE_CLIENT_URLS="https://10.0.4.XXX:2379" ETCD_INITIAL_CLUSTER="etcd121=https://10.10.10.121:2380,etcd122=https://10.10.10.122:2380,etcd123=https://10.10.10.123:2380" ETCD_INITIAL_CLUSTER_TOKEN="fucking-etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new" EOF -
建立 etcd.service
cat /home/deploy/etcd.service <<EOF [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target # [Service] Type=notify EnvironmentFile=/opt/etcd/cfg/etcd ExecStart=/opt/etcd/bin/etcd \ --name=${ETCD_NAME} \ --data-dir=${ETCD_DATA_DIR} \ --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \ --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \ --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \ --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \ --initial-cluster=${ETCD_INITIAL_CLUSTER} \ --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \ --initial-cluster-state=${ETCD_INITIAL_CLUSTER_STATE} \ --cert-file=/opt/etcd/ssl/server.pem \ --key-file=/opt/etcd/ssl/server-key.pem \ --peer-cert-file=/opt/etcd/ssl/server.pem \ --peer-key-file=/opt/etcd/ssl/server-key.pem \ --trusted-ca-file=/opt/etcd/ssl/ca.pem \ --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem Restart=on-failure LimitNOFILE=65536 # [Install] WantedBy=multi-user.target EOF -
部署到遠端三個 etcd 節點(etcd121、etcd122、etcd123)
cd /home/deploy for id in $(seq 121 123); do ip="10.0.4.$id" scp -r etcd $ip:/opt/ ssh $ip "sed -i 's/XXX/$id/g' /opt/etcd/cfg/etcd" scp etcd.service $ip:/usr/lib/systemd/system/ systemctl -H $ip daemon-reload systemctl -H $ip enable etcd" done -
啟動三個 etcd 節點的 etcd 服務
for ip in $(seq -f'10.0.4.%g' 121 123); do systemctl -H $ip start etcd done -
檢視 etcd 叢集狀態
cd /home/deploy/etcd/ssl ../bin/etcdctl \ --ca-file=ca.pem \ --cert-file=server.pem \ --key-file=server-key.pem \ --endpoints="https://10.0.4.121:2379,https://10.0.4.122:2379,https://10.0.4.123:2379" \ cluster-health
安裝 Docker
- 在每個 k8s node 伺服器(node124、node125、node126)上操作下面步驟
- 安裝 docker-ce，參考這裡
部署 Flannel 網路
-
在部署機(master120)上操作下面步驟
-
建立 flannel 部署目錄
cd /home/deploy mkdir flannel/{bin,cfg,ssl} -p # 複製 etcd 證書到 flannel 證書目錄下 rm -rf flannel/ssl/etcd && cp -af etcd/ssl flannel/ssl/etcd -
連線 etcd，寫入預定義的子網段
cd /home/deploy/etcd/ssl ../bin/etcdctl \ --ca-file=ca.pem \ --cert-file=server.pem \ --key-file=server-key.pem \ --endpoints="https://10.0.4.121:2379,https://10.0.4.122:2379,https://10.0.4.123:2379" \ set /coreos.com/network/config '{"Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}' -
獲取 flannel 二進位制包
cd /home/deploy wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz tar zxf flannel-v0.11.0-linux-amd64.tar.gz cp -af flanneld mk-docker-opts.sh flannel/bin/ -
建立 flannel 配置檔案
cat > /home/deploy/flannel/cfg/flanneld <<EOF FLANNEL_OPTIONS=" \ --etcd-endpoints=https://10.0.4.121:2379,https://10.0.4.122:2379,https://10.0.4.123:2379 \ -etcd-cafile=/opt/kubernetes/ssl/etcd/ca.pem \ -etcd-certfile=/opt/kubernetes/ssl/etcd/server.pem \ -etcd-keyfile=/opt/kubernetes/ssl/etcd/server-key.pem \ " EOF -
建立 flanneld.service
cat > /home/deploy/flanneld.service <<EOF [Unit] Description=Flanneld overlay address etcd agent After=network-online.target network.target Before=docker.service # [Service] Type=notify EnvironmentFile=/opt/kubernetes/cfg/flanneld ExecStart=/opt/kubernetes/bin/flanneld --ip-masq $FLANNEL_OPTIONS ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env Restart=on-failure # [Install] WantedBy=multi-user.target EOF -
建立 docker.service，用於從指定的子網段啟動 docker
cat > /home/deploy/docker.service <<EOF [Unit] Description=Docker Application Container Engine Documentation=https://docs.docker.com After=network-online.target firewalld.service Wants=network-online.target # [Service] Type=notify EnvironmentFile=/run/flannel/subnet.env ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS ExecReload=/bin/kill -s HUP $MAINPID LimitNOFILE=infinity LimitNPROC=infinity LimitCORE=infinity TimeoutStartSec=0 Delegate=yes KillMode=process Restart=on-failure StartLimitBurst=3 StartLimitInterval=60s # [Install] WantedBy=multi-user.target EOF -
部署到遠端三個 k8s node 節點(node124、node125、node126)
cd /home/deploy for ip in $(seq -f'10.0.4.%g' 124 126); do systemctl -H $ip stop docker ssh $ip "mkdir /opt/kubernetes" scp -r flannel/* $ip:/opt/kubernetes/ scp flanneld.service docker.service $ip:/usr/lib/systemd/system/ systemctl -H $ip daemon-reload systemctl -H $ip enable flanneld done -
啟動三個 k8s node 節點的 flanneld 和 docker 服務
for ip in $(seq -f'10.0.4.%g' 124 126); do systemctl -H $ip start flanneld systemctl -H $ip start docker done -
啟動完成後，不同節點的 docker0 網絡卡 ip 互通
部署 Master 節點
-
部署前確保前面的 etcd 叢集、flannel 網路和 docekr 都是正常的
-
在部署機(master120，即當前 master 節點)上操作下面步驟
-
建立 master 部署目錄
cd /home/deploy mkdir master/{bin,cfg,ssl} -p # 複製 etcd 證書到 master 證書目錄下 rm -rf master/ssl/etcd && cp -af etcd/ssl master/ssl/etcd -
建立 ca-config.json 檔案
cat > /home/deploy/master/ssl/ca-config.json <<EOF { "signing": { "default": { "expiry": "87600h" }, "profiles": { "kubernetes": { "expiry": "87600h", "usages": [ "signing", "key encipherment", "server auth", "client auth" ] } } } } EOF -
建立 ca-csr.json 檔案
cat > /home/deploy/master/ssl/ca-csr.json <<EOF { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "Beijing", "ST": "Beijing", "O": "k8s", "OU": "System" } ] } EOF -
建立 kube-apiserver-csr.json 檔案
cat > /home/deploy/master/ssl/kube-apiserver-csr.json { "CN": "kubernetes", "hosts": [ "127.0.0.1", "10.0.4.120" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "BeiJing", "ST": "BeiJing", "O": "k8s", "OU": "System" } ] } EOF -
建立 kube-proxy-csr.json 檔案
cat > /home/deploy/master/ssl/kube-proxy-csr.json <<EOF { "CN": "system:kube-proxy", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "BeiJing", "ST": "BeiJing", "O": "k8s", "OU": "System" } ] } EOF -
生成證書
cd /home/deploy/master/ssl cfssl gencert -initca ca-csr.json | cfssljson -bare ca - cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy -
獲取 kubernetes 二進位制包
cd /home/deploy wget https://dl.k8s.io/v1.12.6/kubernetes-server-linux-amd64.tar.gz tar zxf kubernetes-server-linux-amd64.tar.gz cd kubernetes/server/bin scp kube-apiserver kube-scheduler kube-controller-manager kubectl /home/deploy/master/bin/ -
建立 token 檔案
echo '1111222233334444aaaabbbbccccdddd,kubelet-bootstrap,10001,"system:kubelet-bootstrap"' > /home/deploy/master/cfg/token.csv -
建立 kube-apiserver 配置檔案
cat > /home/deploy/master/cfg/kube-apiserver <<EOF KUBE_APISERVER_OPTS=" \ --logtostderr=true \ --v=4 \ --etcd-servers=https://10.0.4.121:2379,https://10.0.4.122:2379,https://10.0.4.123:2379 \ --bind-address=10.0.4.120 \ --secure-port=6443 \ --advertise-address=10.0.4.120 \ --allow-privileged=true \ --service-cluster-ip-range=10.10.9.0/24 \ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,NodeRestriction \ --authorization-mode=RBAC,Node \ --enable-bootstrap-token-auth \ --token-auth-file=/opt/kubernetes/cfg/token.csv \ --service-node-port-range=30000-50000 \ --tls-cert-file=/opt/kubernetes/ssl/kube-apiserver.pem \ --tls-private-key-file=/opt/kubernetes/ssl/kube-apiserver-key.pem \ --client-ca-file=/opt/kubernetes/ssl/ca.pem \ --service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \ --etcd-cafile=/opt/etcd/ssl/etcd/ca.pem \ --etcd-certfile=/opt/etcd/ssl/etcd/server.pem \ --etcd-keyfile=/opt/etcd/ssl/etcd/server-key.pem \ " EOF -
建立 kube-apiserver.service 檔案
cat > /home/deploy/kube-apiserver.service <<EOF [Unit] Description=Kubernetes API Server Documentation=https://github.com/kubernetes/kubernetes # [Service] EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver ExecStart=/opt/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS Restart=on-failure # [Install] WantedBy=multi-user.target EOF -
建立 kube-scheduler 配置檔案
cat > /home/deploy/master/cfg/kube-scheduler <<EOF KUBE_SCHEDULER_OPTS=" \ --logtostderr=true \ --v=4 \ --master=127.0.0.1:8080 \ --leader-elect \ " EOF -
建立 kube-scheduler.service
cat > /home/deploy/kube-scheduler.service <<EOF [Unit] Description=Kubernetes Scheduler Documentation=https://github.com/kubernetes/kubernetes # [Service] EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler ExecStart=/opt/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS Restart=on-failure # [Install] WantedBy=multi-user.target EOF -
建立 kube-controller-mananger 配置檔案
cat > /home/deploy/master/cfg/kube-controller-manager <<EOF KUBE_CONTROLLER_MANAGER_OPTS=" \ --logtostderr=true \ --v=4 \ --master=127.0.0.1:8080 \ --leader-elect=true \ --address=127.0.0.1 \ --service-cluster-ip-range=10.10.9.0/24 \ --cluster-name=my_k8s_cluster \ --cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \ --cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \ --root-ca-file=/opt/kubernetes/ssl/ca.pem \ --service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \ " EOF -
建立 kube-controller-manager.service
cat > /home/deploy/kube-controller-manager.service <<EOF [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/kubernetes/kubernetes # [Service] EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager ExecStart=/opt/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS Restart=on-failure # [Install] WantedBy=multi-user.target EOF -
部署到執行目錄
cd /home/deploy mkdir /opt/kubernetes scp -r master/* /opt/kubernetes/ scp kube-apiserver.service kube-scheduler.service kube-controller-manager /usr/lib/systemd/system/ systemctl daemon-reload systemctl enable kube-apiserver kube-scheduler kube-controller-manager ln -sf /opt/kubernetes/bin/kubectl /usr/local/bin/ -
啟動 k8s master 各元件
systemctl start kube-apiserver systemctl start kube-scheduler systemctl start kube-controller-manager -
檢視叢集各元件狀態
kubectl get cs
部署 k8s node 節點
-
在部署機(master120)上操作下面步驟
-
建立 node 部署目錄
cd /home/deploy mkdir node/{bin,cfg,ssl} -p scp kubernetes/server/bin/{kubelet,kube-proxy} node/bin/ # 複製 master 證書到 node 證書目錄下 scp master/ssl/*.pem node/ssl/ -
生成 bootstrap.kubeconfig 檔案
export BOOTSTRAP_TOKEN=1111222233334444aaaabbbbccccdddd export KUBE_APISERVER="https://10.0.4.120:6443" cd /home/deploy/node/ssl # 設定叢集引數 kubectl config set-cluster my_k8s_cluster \ --certificate-authority=ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=../cfg/bootstrap.kubeconfig # 設定客戶端認證引數 kubectl config set-credentials kubelet-bootstrap \ --token=${BOOTSTRAP_TOKEN} \ --kubeconfig=../cfg/bootstrap.kubeconfig # 設定上下文引數 kubectl config set-context default \ --cluster=my_k8s_cluster \ --user=kubelet-bootstrap \ --kubeconfig=../cfg/bootstrap.kubeconfig # 設定預設上下文 kubectl config use-context default --kubeconfig=../cfg/bootstrap.kubeconfig -
生成 kube-proxy.kubeconfig 檔案
export BOOTSTRAP_TOKEN=1111222233334444aaaabbbbccccdddd export KUBE_APISERVER="https://10.0.4.120:6443" cd /home/deploy/node/ssl kubectl config set-cluster my_k8s_cluster \ --certificate-authority=ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=../cfg/kube-proxy.kubeconfig kubectl config set-credentials kube-proxy \ --client-certificate=kube-proxy.pem \ --client-key=kube-proxy-key.pem \ --embed-certs=true \ --kubeconfig=../cfg/kube-proxy.kubeconfig kubectl config set-context default \ --cluster=my_k8s_cluster \ --user=kube-proxy \ --kubeconfig=../cfg/kube-proxy.kubeconfig kubectl config use-context default --kubeconfig=../cfg/kube-proxy.kubeconfig -
建立 kubelet 配置檔案
cat > /home/deploy/node/cfg/kubelet <<EOF KUBELET_OPTS=" \ --logtostderr=true \ --v=4 \ --hostname-override=10.0.4.XXX \ --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \ --bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \ --config=/opt/kubernetes/cfg/kubelet.config \ --cert-dir=/opt/kubernetes/ssl \ --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 \ " EOF # registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 該映象可以提前導本地入區域網中的私有 docker 倉庫中 -
建立 kubelet.config 配置檔案
cat > /home/deploy/node/cfg/kubelet.config <<EOF kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 address: 10.0.4.XXX port: 10250 readOnlyPort: 10255 cgroupDriver: cgroupfs clusterDNS: ["10.10.9.2"] clusterDomain: cluster.local. failSwapOn: false authentication: anonymous: enabled: true EOF -
建立 kubelet.service
cat > /home/deploy/kubelet.service <<EOF [Unit] Description=Kubernetes Kubelet After=docker.service Requires=docker.service # [Service] EnvironmentFile=/opt/kubernetes/cfg/kubelet ExecStart=/opt/kubernetes/bin/kubelet $KUBELET_OPTS Restart=on-failure KillMode=process # [Install] WantedBy=multi-user.target EOF -
建立 kube-proxy 配置檔案
cat > /home/deploy/node/cfg/kube-proxy <<EOF KUBE_PROXY_OPTS=" \ --logtostderr=true \ --v=4 \ --hostname-override=10.0.4.XXX \ --cluster-cidr=10.10.9.0/24 \ --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig \ " EOF -
建立 kube-proxy.service
cat > /home/deploy/kube-proxy.service <<EOF [Unit] Description=Kubernetes Proxy After=network.target # [Service] EnvironmentFile=-/opt/kubernetes/cfg/kube-proxy ExecStart=/opt/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS Restart=on-failure # [Install] WantedBy=multi-user.target EOF -
部署到遠端三個 k8s node 節點(node124、node125、node126)
cd /home/deploy for id in $(seq 124 126); do ip="10.0.4.$id" scp -r node/* $ip:/opt/kubernetes/ ssh $ip "sed -i 's/XXX/$id/g' /opt/kubernetes/cfg/kubelet" ssh $ip "sed -i 's/XXX/$id/g' /opt/kubernetes/cfg/kubelet.config" ssh $ip "sed -i 's/XXX/$id/g' /opt/kubernetes/cfg/kube-proxy" scp kubelet.service kube-proxy.service $ip:/usr/lib/systemd/system/ systemctl -H $ip daemon-reload systemctl -H $ip enable kubelet kube-proxy" done -
啟動三個 k8s node 節點的 kubelet 和 kube-proxy 服務
for ip in $(seq -f'10.0.4.%g' 124 126); do systemctl -H $ip start kubelet systemctl -H $ip start kube-proxy done -
檢視叢集狀態
kubectl get node kubectl get cs