shell install 3 nodes k8s
阿新 • • 發佈：2018-12-30
# change time zone cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime timedatectl set-timezone Asia/Shanghai rm /etc/yum.repos.d/CentOS-Base.repo cp /vagrant/yum/*.* /etc/yum.repos.d/ mv /etc/yum.repos.d/CentOS7-Base-163.repo /etc/yum.repos.d/CentOS-Base.repo echo 'disable selinux' setenforce 0 sed -i 's/=enforcing/=disabled/g' /etc/selinux/config systemctl stop firewalld systemctl disable firewalld echo "關閉 swap 分割槽" swapoff -a sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab echo "關閉 SELinux" echo "install sshd" rm -rf /etc/ssh/sshd_config yum -y update openssh-server rpm -e --nodeps -f openssh-server yum -y update openssh-server yum -y install openssh-server echo "start sshd" systemctl restart sshd echo "安裝依賴包" yum install -y epel-release yum install -y conntrack ipvsadm ipset jq sysstat curl iptables libseccomp echo "set iptables" iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat iptables -P FORWARD ACCEPT echo "install wget" yum -y install wget echo 'sync time' yum -y install ntp systemctl start ntpd systemctl enable ntpd echo "add user k8s" useradd -m k8s echo "set password" sh -c 'echo 123456 | passwd k8s --stdin' # 為 k8s 賬戶設定密碼 echo "add k8s to group wheel" gpasswd -a k8s wheel echo "add user docker " useradd -m docker echo "add k8s to group docker" gpasswd -a k8s docker echo "new dir" mkdir -p /etc/docker/ echo "建立目錄" mkdir -p /opt/k8s/bin chown -R k8s /opt/k8s mkdir -p /etc/kubernetes/cert chown -R k8s /etc/kubernetes mkdir -p /etc/etcd/cert chown -R k8s /etc/etcd/cert mkdir -p /var/lib/etcd && chown -R k8s /etc/etcd/cert #######################single-mode env-config-file################# echo "store environment.sh " cp /vagrant/cluster-environment.sh /opt/k8s/bin/ source /opt/k8s/bin/cluster-environment.sh # define upload file to file server function uploadFiles(){ arr=$1 for fileName in ${arr[*]} do _file=$2/$fileName curl -F "subDirName=$3" -F "[email protected]$_file" ${FILE_UPLOAD_URL} done } # define download file from file server function downloadFiles(){ arr=$1 for str in ${arr[*]} do echo $str wget -O ${str} ${FILE_DOWNLOAD_URL}${str}?subDirName=$3 mv ${str} $2 done } function downloadFile(){ wget -O $1 ${FILE_DOWNLOAD_URL}$1?subDirName=$2 } cat >> /etc/hosts <<EOF 172.27.130.105 cluster-node1 172.27.130.111 cluster-node2 172.27.130.112 cluster-node3 EOF echo "將可執行檔案路徑 /opt/k8s/bin 新增到 PATH 變數中" sh -c "echo 'PATH=/opt/k8s/bin:$PATH:$HOME/bin:$JAVA_HOME/bin' >>/root/.bashrc" echo 'PATH=/opt/k8s/bin:$PATH:$HOME/bin:$JAVA_HOME/bin' >>~/.bashrc echo "k8s user add evn path" sh -c "echo 'PATH=/opt/k8s/bin:$PATH:$HOME/bin:$JAVA_HOME/bin' >>/home/k8s/.bashrc" echo 'PATH=/opt/k8s/bin:$PATH:$HOME/bin:$JAVA_HOME/bin' >>/home/k8s/.bashrc echo "載入核心模組" modprobe br_netfilter modprobe ip_vs echo "設定系統引數" downloadFile kubernetes.conf basic-config cp kubernetes.conf /etc/sysctl.d/kubernetes.conf sysctl -p /etc/sysctl.d/kubernetes.conf mount -t cgroup -o cpu,cpuacct none /sys/fs/cgroup/cpu,cpuacct #########################02.建立 CA 證書和祕鑰############################## echo "#########################02.建立 CA 證書和祕鑰##############################" if [ $1 -eq 1 ];then echo "安裝 cfssl 工具集" mkdir -p /opt/k8s/cert && chown -R k8s /opt/k8s && cd /opt/k8s mv /vagrant/cfssl/cfssl_linux-amd64 /opt/k8s/bin/cfssl mv /vagrant/cfssl/cfssljson_linux-amd64 /opt/k8s/bin/cfssljson mv /vagrant/cfssl/cfssl-certinfo_linux-amd64 /opt/k8s/bin/cfssl-certinfo chmod +x /opt/k8s/bin/* export PATH=/opt/k8s/bin:$PATH echo "建立根證書 (CA)" downloadFile ca-config.json basic-config echo "建立證書籤名請求檔案" downloadFile ca-csr.json basic-config echo "生成 CA 證書和私鑰" cfssl gencert -initca ca-csr.json | cfssljson -bare ca ls ca* mkdir -p /etc/kubernetes/cert && chown -R k8s /etc/kubernetes cp ca*.pem ca-config.json /etc/kubernetes/cert uploadFiles "${CA_FILES[*]}" /etc/kubernetes/cert ${CA} else echo "give out to node2 and node3" mkdir -p /etc/kubernetes/cert && chown -R k8s /etc/kubernetes downloadFiles "${CA_FILES[*]}" /etc/kubernetes/cert ${CA} fi ################03.部署 kubectl 命令列工具 ALL install ############# echo "################03.部署 kubectl 命令列工具 ALL install #############" #!important use absolute path cd /vagrant/kubernetes echo "cp /vagrant/kubernetes/client/bin/kubectl /opt/k8s/bin/" cp /vagrant/kubernetes/client/bin/kubectl /opt/k8s/bin/ echo "chmod +x /opt/k8s/bin/*" chmod +x /opt/k8s/bin/* echo "建立證書籤名請求============" if [ $1 -eq 1 ];then echo "建立證書籤名請求" downloadFile admin-csr.json basic-config echo "生成證書和私鑰" cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \ -ca-key=/etc/kubernetes/cert/ca-key.pem \ -config=/etc/kubernetes/cert/ca-config.json \ -profile=kubernetes admin-csr.json | cfssljson -bare admin ls admin* echo "設定叢集引數" kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/cert/ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kubectl.kubeconfig echo "設定客戶端認證引數" kubectl config set-credentials admin \ --client-certificate=admin.pem \ --client-key=admin-key.pem \ --embed-certs=true \ --kubeconfig=kubectl.kubeconfig echo "設定上下文引數" kubectl config set-context kubernetes \ --cluster=kubernetes \ --user=admin \ --kubeconfig=kubectl.kubeconfig echo "設定預設上下文" kubectl config use-context kubernetes --kubeconfig=kubectl.kubeconfig echo "make dir /.kube" #ll -a #!problem mkdir -p /home/k8s/.kube cp kubectl.kubeconfig /home/k8s/.kube/config mkdir -p /root/.kube cp kubectl.kubeconfig /root/.kube/config echo "put kubectl.kubeconfig to dir /vagrant/kubectl-config for node2,node3" uploadFiles "${CTL_CONF_FILES[*]}" /root/.kube/ ${CTL_CONF} else mkdir -p /home/k8s/.kube mkdir -p /root/.kube downloadFiles "${CTL_CONF_FILES[*]}" /home/k8s/.kube/ ${CTL_CONF} downloadFiles "${CTL_CONF_FILES[*]}" /root/.kube/ ${CTL_CONF} fi #################04.部署 etcd 叢集######################## echo "#################04.部署 etcd 叢集########################" #etcd-v3.3.7-linux-amd64 cp /vagrant/etcd-v3.3.7-linux-amd64/etcd* /opt/k8s/bin chmod +x /opt/k8s/bin/* if [ $1 -eq 1 ];then downloadFile etcd-csr.json basic-config echo "生成證書和私鑰" cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \ -ca-key=/etc/kubernetes/cert/ca-key.pem \ -config=/etc/kubernetes/cert/ca-config.json \ -profile=kubernetes etcd-csr.json | cfssljson -bare etcd ls etcd* echo "生成的證書和私鑰" mkdir -p /etc/etcd/cert && chown -R k8s /etc/etcd/cert cp etcd*.pem /etc/etcd/cert/ uploadFiles "${ETCD_CA_FILES[*]}" /etc/etcd/cert/ ${ETCD_CA} else mkdir -p /etc/etcd/cert && chown -R k8s /etc/etcd/cert downloadFiles "${ETCD_CA_FILES[*]}" /etc/etcd/cert/ ${ETCD_CA} fi echo "ips = ${ETCD_NODES}" downloadFile etcd.service.template basic-config idx=`expr $1 - 1` echo "建立 systemd unit 檔案" sed -e "s/##NODE_NAME##/${NODE_NAMES[$idx]}/" -e "s/##NODE_IP##/${NODE_IPS[$idx]}/" -e "s|##ETCD_NODES##|${ETCD_NODES}|" etcd.service.template > etcd-${NODE_IPS[$idx]}.service ls *.service mkdir -p /var/lib/etcd && chown -R k8s /var/lib/etcd cp etcd-${NODE_IPS[$idx]}.service /etc/systemd/system/etcd.service echo "!important add execute permission" chown -R k8s /etc/etcd/cert/ chmod +x -R /etc/etcd/cert/ #!important chown -R k8s /etc/kubernetes/cert/ chmod -R +x /etc/kubernetes/cert cat /etc/systemd/system/etcd.service echo "start etcd" systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd & echo "etcd status" systemctl status etcd|grep Active echo "look ectd log" journalctl -u etcd #################05.部署 flannel 網路######################### echo "#################05.部署 flannel 網路#########################" mkdir /vagrant/flannel echo "tar flannel.." echo "copy to path" cp /vagrant/flannel/{flanneld,mk-docker-opts.sh} /opt/k8s/bin/ echo "add execute permission.." chmod +x /opt/k8s/bin/* if [ $1 -eq 1 ];then echo "建立證書籤名請求" downloadFile flanneld-csr.json basic-config echo "生成證書和私鑰" cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \ -ca-key=/etc/kubernetes/cert/ca-key.pem \ -config=/etc/kubernetes/cert/ca-config.json \ -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld ls flanneld*pem echo "將生成的證書和私鑰分發到所有節點" mkdir -p /etc/flanneld/cert && chown -R k8s /etc/flanneld cp flanneld*.pem /etc/flanneld/cert uploadFiles "${FLAN_CA_FILES[*]}" /etc/flanneld/cert ${FLAN_CA} echo "向 etcd 寫入叢集 Pod 網段資訊[email protected]" etcdctl \ --endpoints=${ETCD_ENDPOINTS} \ --ca-file=/etc/kubernetes/cert/ca.pem \ --cert-file=/etc/flanneld/cert/flanneld.pem \ --key-file=/etc/flanneld/cert/flanneld-key.pem \ set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}' else mkdir -p /etc/flanneld/cert && chown -R k8s /etc/flanneld downloadFiles "${FLAN_CA_FILES[*]}" /etc/flanneld/cert ${FLAN_CA} fi echo "建立 flanneld 的 systemd unit 檔案" #eth0 export IFACE=enp0s8 downloadFile flanneld.service.template basic-config sed -e "s|##ETCD_ENDPOINTS##|${ETCD_ENDPOINTS}|" -e "s|##FLANNEL_ETCD_PREFIX##|${FLANNEL_ETCD_PREFIX}|" -e "s/##IFACE##/${IFACE}/" flanneld.service.template > flanneld.service echo "分發 flanneld systemd unit 檔案到所有節點" cp flanneld.service /etc/systemd/system/ echo "啟動 flanneld 服務" systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld echo "檢查啟動結果" systemctl status flanneld|grep Active echo "檢視日誌" journalctl -u flanneld ################06-0.部署 master 節點################# echo "################06-0.部署 master 節點#################" echo "install k8s " cd /vagrant/kubernetes echo "拷貝到所有 master 節點" cp /vagrant/kubernetes/server/bin/* /opt/k8s/bin/ chmod +x /opt/k8s/bin/* ################06-1.部署高可用元件#################### echo "################06-1.部署高可用元件####################" echo "安裝軟體包 keepalived haproxy" yum install -y keepalived haproxy if [ $1 -eq 1 ];then echo "haproxy 配置檔案" downloadFile haproxy.cfg basic-config echo "下發 haproxy.cfg 到所有 master 節點" cp haproxy.cfg /etc/haproxy else downloadFile haproxy.cfg basic-config mv haproxy.cfg /etc/haproxy fi echo "起 haproxy 服務" systemctl restart haproxy echo "檢查 haproxy 服務狀態" systemctl status haproxy|grep Active echo "檢視日誌" journalctl -u haproxy if [ $1 -eq 1 ];then echo "keepalived conf file" downloadFile keepalived-master.conf.template basic-config sed -e "s/##VIP_IF##/${VIP_IF}/" -e "s/##MASTER_VIP##/${MASTER_VIP}/" keepalived-master.conf.template > keepalived-master.conf echo "下發 keepalived 配置檔案" cp keepalived-master.conf /etc/keepalived/keepalived.conf else echo "backup 配置檔案" downloadFile keepalived-backup.conf.template basic-config sed -e "s/##VIP_IF##/${VIP_IF}/" -e "s/##MASTER_VIP##/${MASTER_VIP}/" keepalived-backup.conf.template > keepalived-backup.conf cp keepalived-backup.conf /etc/keepalived/keepalived.conf fi echo "起 keepalived 服務" systemctl restart keepalived echo "檢查 keepalived 服務" systemctl status keepalived|grep Active echo "檢視日誌" journalctl -u keepalived echo "ping 通 VIP" /usr/sbin/ip addr show ${VIP_IF} ping -c 1 ${MASTER_VIP} ##################06-2.部署 kube-apiserver 元件############################## if [ $1 -eq 1 ];then echo "##################06-2.部署 kube-apiserver 元件##############################" echo "建立證書籤名請求" downloadFile kubernetes-csr.json.template basic-config sed -e "s/##MASTER_VIP##/${MASTER_VIP}/" -e "s/##CLUSTER_KUBERNETES_SVC_IP##/${CLUSTER_KUBERNETES_SVC_IP}/" kubernetes-csr.json.template > kubernetes-csr.json echo "生成證書和私鑰" cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \ -ca-key=/etc/kubernetes/cert/ca-key.pem \ -config=/etc/kubernetes/cert/ca-config.json \ -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes ls kubernetes*pem echo "將生成的證書和私鑰檔案拷貝到 master 節點" cp kubernetes*.pem /etc/kubernetes/cert/ uploadFiles "${API_CA_FILES[*]}" /etc/kubernetes/cert/ ${API_CA} else downloadFiles "${API_CA_FILES[*]}" /etc/kubernetes/cert/ ${API_CA} fi echo "建立加密配置檔案" downloadFile encryption-config.yaml.template basic-config sed -e "s/##ENCRYPTION_KEY##/${ENCRYPTION_KEY}/" encryption-config.yaml.template > encryption-config.yaml echo "將加密配置檔案拷貝到 master 節點的 /etc/kubernetes 目錄下" cp encryption-config.yaml /etc/kubernetes/ echo "建立 kube-apiserver systemd unit 模板檔案" downloadFile kube-apiserver.service.template basic-config echo "替換模板檔案中的變數，為各節點建立 systemd unit 檔案" idx=`expr $1 - 1` sed -e "s/##NODE_NAME##/${NODE_NAMES[$idx]}/" -e "s/##NODE_IP##/${NODE_IPS[$idx]}/" -e "s|##SERVICE_CIDR##|${SERVICE_CIDR}|" -e "s|##NODE_PORT_RANGE##|${NODE_PORT_RANGE}|" -e "s|##ETCD_ENDPOINTS##|${ETCD_ENDPOINTS}|" kube-apiserver.service.template > kube-apiserver-${NODE_IPS[$idx]}.service ls kube-apiserver*.service echo "分發生成的 systemd unit 檔案" mkdir -p /var/log/kubernetes && chown -R k8s /var/log/kubernetes cp kube-apiserver-${NODE_IPS[$idx]}.service /etc/systemd/system/kube-apiserver.service echo "give permission to exeucte" chown -R k8s /etc/kubernetes/cert/ chmod -R +x /etc/kubernetes/cert echo "啟動 kube-apiserver 服務" systemctl daemon-reload && systemctl enable kube-apiserver && systemctl restart kube-apiserver echo "檢查 kube-apiserver 執行狀態" systemctl status kube-apiserver |grep Active echo " master 節點檢視日誌" journalctl -u kube-apiserver ETCDCTL_API=3 etcdctl \ --endpoints=${ETCD_ENDPOINTS} \ --cacert=/etc/kubernetes/cert/ca.pem \ --cert=/etc/etcd/cert/etcd.pem \ --key=/etc/etcd/cert/etcd-key.pem \ get /registry/ --prefix --keys-only echo "檢查叢集資訊" kubectl cluster-info kubectl get all --all-namespaces kubectl get componentstatuses #echo "檢查 kube-apiserver 監聽的埠" #netstat -lnpt|grep kube echo "授予 kubernetes 證書訪問 kubelet API 的許可權" kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes ###################06-3.部署高可用 kube-controller-manager 叢集################ echo "###################06-3.部署高可用 kube-controller-manager 叢集################" if [ $1 -eq 1 ];then echo "建立 kube-controller-manager 證書和私鑰" echo "single node mode" downloadFile kube-controller-manager-csr.json basic-config echo "生成證書和私鑰" cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \ -ca-key=/etc/kubernetes/cert/ca-key.pem \ -config=/etc/kubernetes/cert/ca-config.json \ -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager echo "copy pem files to cert dir" cp kube-controller-manager*.pem /etc/kubernetes/cert/ chmod +x /etc/kubernetes/cert/*.pem uploadFiles "${CM_CA_FILES[*]}" /etc/kubernetes/cert/ ${CM_CA} else downloadFiles "${CM_CA_FILES[*]}" /etc/kubernetes/cert/ ${CM_CA} chmod +x /etc/kubernetes/cert/*.pem fi if [ $1 -eq 1 ];then echo "建立和分發 kubeconfig 檔案" kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/cert/ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kube-controller-manager.kubeconfig kubectl config set-credentials system:kube-controller-manager \ --client-certificate=/etc/kubernetes/cert/kube-controller-manager.pem \ --client-key=/etc/kubernetes/cert/kube-controller-manager-key.pem \ --embed-certs=true \ --kubeconfig=kube-controller-manager.kubeconfig kubectl config set-context system:kube-controller-manager \ --cluster=kubernetes \ --user=system:kube-controller-manager \ --kubeconfig=kube-controller-manager.kubeconfig kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig echo "分發 kubeconfig 到所有 master 節點" cp kube-controller-manager.kubeconfig /etc/kubernetes/ uploadFiles "${CM_CONF_FILES[*]}" /etc/kubernetes/ ${CM_CONF} else downloadFiles "${CM_CONF_FILES[*]}" /etc/kubernetes/ ${CM_CONF} fi echo "建立和分發 kube-controller-manager systemd unit 檔案" downloadFile kube-controller-manager.service.template basic-config sed -e "s|##SERVICE_CIDR##|${SERVICE_CIDR}|" kube-controller-manager.service.template > kube-controller-manager.service echo "分發 systemd unit 檔案到所有 master 節點" cp kube-controller-manager.service /etc/systemd/system/ #!important chown -R k8s /etc/kubernetes/cert/ chmod -R +x /etc/kubernetes/cert chown -R k8s /etc/kubernetes chmod -R +x /etc/kubernetes echo "啟動 kube-controller-manager 服務" systemctl daemon-reload && systemctl enable kube-controller-manager && systemctl restart kube-controller-manager echo "檢查服務執行狀態" systemctl status kube-controller-manager|grep Active echo "檢視日誌" journalctl -u kube-controller-manager echo "檢視輸出的 metric" curl -s --cacert /etc/kubernetes/cert/ca.pem https://127.0.0.1:10252/metrics |head echo "測試 kube-controller-manager 叢集的高可用===檢視當前的 leader" kubectl get endpoints kube-controller-manager --namespace=kube-system -o yaml #######################06-4.部署高可用 kube-scheduler 叢集###################### echo "#######################06-4.部署高可用 kube-scheduler 叢集######################" if [ $1 -eq 1 ];then echo "建立 kube-scheduler 證書和私鑰" downloadFile kube-scheduler-csr.json basic-config echo "generate scheduler ca" cfssl gencert \ -ca=/etc/kubernetes/cert/ca.pem \ -ca-key=/etc/kubernetes/cert/ca-key.pem \ -config=/etc/kubernetes/cert/ca-config.json \ -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler echo "copy pem files to cert dir" cp kube-scheduler*.pem /etc/kubernetes/cert/ chmod +x /etc/kubernetes/cert/*.pem echo "建立和分發 kubeconfig 檔案" kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/cert/ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kube-scheduler.kubeconfig kubectl config set-credentials system:kube-scheduler \ --client-certificate=kube-scheduler.pem \ --client-key=kube-scheduler-key.pem \ --embed-certs=true \ --kubeconfig=kube-scheduler.kubeconfig kubectl config set-context system:kube-scheduler \ --cluster=kubernetes \ --user=system:kube-scheduler \ --kubeconfig=kube-scheduler.kubeconfig kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig echo "分發 kubeconfig 到所有 master 節點" cp kube-scheduler.kubeconfig /etc/kubernetes/ uploadFiles "${SCH_CONF_FILES[*]}" /etc/kubernetes ${SCH_CONF} else downloadFiles "${SCH_CONF_FILES[*]}" /etc/kubernetes ${SCH_CONF} fi echo "建立和分發 kube-scheduler systemd unit 檔案" downloadFile kube-scheduler.service basic-config echo "分發 systemd unit 檔案到所有 master 節點" cp kube-scheduler.service /etc/systemd/system/ #!important chown -R k8s /etc/kubernetes/cert/ chmod -R +x /etc/kubernetes/cert chown -R k8s /etc/kubernetes chmod -R +x /etc/kubernetes echo "啟動 kube-scheduler 服務" systemctl daemon-reload && systemctl enable kube-scheduler && systemctl restart kube-scheduler echo "檢查服務執行狀態" systemctl status kube-scheduler|grep Active journalctl -u kube-scheduler echo "檢視輸出的 metric" curl -s http://127.0.0.1:10251/metrics |head echo "測試 kube-scheduler 叢集的高可用====檢視當前的 leader" kubectl get endpoints kube-scheduler --namespace=kube-system -o yaml ######################install client#################################### echo "######################install client####################################" echo "#################### 部署 docker ###########################" #tar -xzvf /vagrant/docker-18.03.1-ce.tgz -C /vagrant #cp /vagrant/docker/docker* /opt/k8s/bin/ echo "install the latest version of docker" yum install -y yum-utils \ device-mapper-persistent-data \ lvm2 yum-config-manager \ --add-repo \ https://download.docker.com/linux/centos/docker-ce.repo yum -y install docker-ce rm -f /opt/k8s/bin/docker* cp /usr/bin/docker* /opt/k8s/bin chmod +x /opt/k8s/bin/* echo "建立和分發docker systemd unit 檔案" downloadFile docker.service basic-config cp docker.service /etc/systemd/system/ echo "append mirror to daemon" downloadFile docker-daemon.json basic-config mv docker-daemon.json /etc/docker/daemon.json echo "start docker service" systemctl daemon-reload && systemctl enable docker && systemctl restart docker sysctl -p /etc/sysctl.d/kubernetes.conf echo "check docker status" systemctl status docker|grep Active echo "check docker log" journalctl -u docker echo "檢查 docker docker0 網橋" ip addr show flannel.1 && /usr/sbin/ip addr show docker0 ######################07-2.部署 kubelet 元件###################### echo "######################07-2.部署 kubelet 元件######################" echo "建立 kubelet bootstrap kubeconfig 檔案" echo "# 建立 token" idx=`expr $1 - 1` export BOOTSTRAP_TOKEN=$(kubeadm token create \ --description kubelet-bootstrap-token \ --groups system:bootstrappers:${NODE_NAMES[$idx]} \ --kubeconfig ~/.kube/config) echo "# 設定叢集引數" kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/cert/ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig echo "# 設定客戶端認證引數" kubectl config set-credentials kubelet-bootstrap \ --token=${BOOTSTRAP_TOKEN} \ --kubeconfig=kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig echo "# 設定上下文引數" kubectl config set-context default \ --cluster=kubernetes \ --user=kubelet-bootstrap \ --kubeconfig=kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig echo "# 設定預設上下文" kubectl config use-context default --kubeconfig=kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig echo "檢視 kubeadm 為各節點建立的 token" kubeadm token list --kubeconfig ~/.kube/config echo "各 token 關聯的 Secret" kubectl get secrets -n kube-system echo "分發 bootstrap kubeconfig 檔案到所有 worker 節點" cp kubelet-bootstrap-${NODE_NAMES[$idx]}.kubeconfig /etc/kubernetes/kubelet-bootstrap.kubeconfig echo "!important k8s.V1.8 not use this file" echo "建立和分發 kubelet 引數配置檔案" downloadFile kubelet.config.json.template basic-config echo "為各節點建立和分發 kubelet 配置檔案" idx=`expr $1 - 1` sed -e "s/##NODE_IP##/${NODE_IPS[$idx]}/" -e "s/##CLUSTER_DNS_DOMAIN##/${CLUSTER_DNS_DOMAIN}/" -e "s/##CLUSTER_DNS_SVC_IP##/${CLUSTER_DNS_SVC_IP}/" kubelet.config.json.template > kubelet.config-${NODE_IPS[$idx]}.json cp kubelet.config-${NODE_IPS[$idx]}.json /etc/kubernetes/kubelet.config.json echo "建立和分發 kubelet systemd unit 檔案" downloadFile kubelet.service.template basic-config echo "為各節點建立和分發 kubelet systemd unit 檔案" idx=`expr $1 - 1` sed -e "s/##NODE_NAME##/${NODE_NAMES[$idx]}/" kubelet.service.template > kubelet-${NODE_NAMES[$idx]}.service cp kubelet-${NODE_NAMES[$idx]}.service /etc/systemd/system/kubelet.service cp /vagrant/kubelet.kubeconfig /etc/kubernetes/ echo "Bootstrap Token Auth 和授予許可權" journalctl -u kubelet -a |grep -A 2 'certificatesigningrequests' kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --group=system:bootstrappers echo "啟動 kubelet 服務" chown -R k8s /etc/kubernetes/ chmod -R +x /etc/kubernetes/ mkdir -p /var/lib/kubelet systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet kubectl get csr kubectl get nodes echo "approve kubelet CSR 請求" echo "檢視 CSR 列表" kubectl get csr kubectl describe csr node-csr-QzuuQiuUfcSdp3j5W4B2UOuvQ_n9aTNHAlrLzVFiqrk echo "自動 approve CSR 請求" downloadFile csr-crb.yaml basic-config kubectl apply -f csr-crb.yaml kubectl get csr kubectl get nodes ls -l /etc/kubernetes/kubelet.kubeconfig ls -l /etc/kubernetes/cert/|grep kubelet ######################07-3.部署 kube-proxy 元件################################### echo "建立 kube-proxy 證書" downloadFile kube-proxy-csr.json basic-config echo "生成證書和私鑰" cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \ -ca-key=/etc/kubernetes/cert/ca-key.pem \ -config=/etc/kubernetes/cert/ca-config.json \ -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy echo "建立和分發 kubeconfig 檔案" kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/cert/ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kube-proxy.kubeconfig kubectl config set-credentials kube-proxy \ --client-certificate=kube-proxy.pem \ --client-key=kube-proxy-key.pem \ --embed-certs=true \ --kubeconfig=kube-proxy.kubeconfig kubectl config set-context default \ --cluster=kubernetes \ --user=kube-proxy \ --kubeconfig=kube-proxy.kubeconfig kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig echo "分發 kubeconfig 檔案!important chown -R " cp kube-proxy.kubeconfig /etc/kubernetes/ echo "建立 kube-proxy 配置檔案" downloadFile kube-proxy.config.yaml.template basic-config idx=`expr $1 - 1` sed -e "s/##NODE_NAME##/${NODE_NAMES[$idx]}/" -e "s/##NODE_IP##/${NODE_IPS[$idx]}/" -e "s|##CLUSTER_CIDR##|${CLUSTER_CIDR}|" kube-proxy.config.yaml.template > kube-proxy-${NODE_NAMES[$idx]}.config.yaml cp kube-proxy-${NODE_NAMES[$idx]}.config.yaml /etc/kubernetes/kube-proxy.config.yaml echo "建立和分發 kube-proxy systemd unit 檔案" downloadFile kube-proxy.service basic-config echo "分發 kube-proxy systemd unit 檔案" cp kube-proxy.service /etc/systemd/system/ chown -R k8s /etc/kubernetes/ chmod -R +x /etc/kubernetes/ echo "啟動 kube-proxy 服務" mkdir -p /var/lib/kube-proxy mkdir -p /var/log/kubernetes && chown -R k8s /var/log/kubernetes systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy echo "檢查啟動結果" systemctl status kube-proxy|grep Active journalctl -u kube-proxy