# master01上執行
USER=root # customizable
CONTROL_PLANE_IPS="10.3.65.18"
for host in ${CONTROL_PLANE_IPS}; do
    scp -P 10088 /etc/kubernetes/pki/ca.crt "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/ca.key "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/sa.key "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/sa.pub "
 
${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:/tmp
    scp -P 10088 /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:/tmp/etcd-ca.crt
    # Quote this line if you are using external etcd
    scp -P 10088 /etc/kubernetes/pki/etcd/ca.key "
 
${USER}"@$host:/tmp/etcd-ca.key
done

#其餘master上執行
USER=tmp # customizable
mkdir -p /etc/kubernetes/pki/etcd
mv /${USER}/ca.crt /etc/kubernetes/pki/
mv /${USER}/ca.key /etc/kubernetes/pki/
mv /${USER}/sa.pub /etc/kubernetes/pki/
mv /${USER}/sa.key /etc/kubernetes/pki/
mv /${USER}/front-proxy-ca.crt /etc/kubernetes/pki/
mv /${USER}/front-proxy-ca.key /etc/kubernetes/pki/
mv
 
 /${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt
# Quote this line if you are using external etcd
mv /${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key

kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers

docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.17.17 k8s.gcr.io/kube-proxy:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-apiserver:v1.17.17 k8s.gcr.io/kube-apiserver:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-controller-manager:v1.17.17 k8s.gcr.io/kube-controller-manager:v1.17.17
docker tag registry.aliyuncs.com/google_containers/kube-scheduler:v1.17.17 k8s.gcr.io/kube-scheduler:v1.17.17
docker tag registry.aliyuncs.com/google_containers/coredns:1.6.5 k8s.gcr.io/coredns:1.6.5
docker tag registry.aliyuncs.com/google_containers/etcd:3.4.3-0 k8s.gcr.io/etcd:3.4.3-0
docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1


#node加入命令
#kubeadm join 10.3.65.37:6443 --token m40u64.k37bumocdfo4gmjj     --discovery-token-ca-cert-hash sha256:d11a8901ec1459035cbd9265d6421438a16af110b506929a4a504809dd0676cf  

#master加入命令

kubeadm join 10.3.65.37:6443 --token e9kub4.1y1tsheeinjonyaz \ --discovery-token-ca-cert-hash sha256:d11a8901ec1459035cbd9265d6421438a16af110b506929a4a504809dd0676cf --control-plane

初始化指令碼，就那幾個生成配置檔案的，需要確認重改，別的都對

#master上操作
scp /etc/yum.repos.d/kubernetes.repo xgcloud-ops-k8s-cluster-3:/etc/yum.repos.d/
scp /etc/yum.repos.d/kubernetes.repo xgcloud-ops-k8s-cluster-2:/etc/yum.repos.d/
scp /etc/yum.repos.d/kubernetes.repo xgcloud-ops-k8s-cluster-1:/etc/yum.repos.d/






#node 上操作
#有網際網路可以用chronyd服務，或者自己的NTP服務
systemctl status chronyd
systemctl restart chronyd

#加host
#hostnamectl set-hostname k8s-master（192.168.73.138主機打命令）
#hostnamectl set-hostname k8s-node01（192.168.73.139主機打命令）
#hostnamectl set-hostname k8s-node02 (192.168.73.140主機打命令)
cat >> /etc/hosts << EOF
10.3.65.37 xgcloud-ops-k8s-cluster-4
10.3.65.18 xgcloud-ops-k8s-cluster-3
10.3.65.14 xgcloud-ops-k8s-cluster-2
10.3.65.48 xgcloud-ops-k8s-cluster-1
EOF
 
#關閉iptables和firewalld服務

systemctl status firewalld
systemctl stop firewalld
systemctl stop iptables
systemctl disable firewalld
systemctl disable iptables

#核心引數永久修改
cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl -p /etc/sysctl.d/k8s.conf

#SELINUX disable
getenforce
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config  && setenforce 0

#禁用swap 
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab 

#啟用ipvs核心模組 寫進去吧，這樣出錯了

cat >> /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for mod in $(ls $ipvs_mods_dir|grep -o "^[^.]*");do
    /sbin/modinfo -F filename $mod &> /dev/null
    if [ $? -eq 0 ];then
        /sbin/modprobe $mod
    fi
done
EOF
###vi /etc/sysconfig/modules/ipvs.modules
###
####!/bin/bash
###ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
###for mod in $(ls $ipvs_mods_dir|grep -o "^[^.]*");do
###    /sbin/modinfo -F filename $mod &> /dev/null
###    if [$? -eq 0 ];then
###        /sbin/modprobe $mod
###    fi
###done

chmod +x /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules


######安裝docker###########
cd /etc/yum.repos.d/
wget http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#scp到各個機器
yum install -y yum-utils   device-mapper-persistent-data   lvm2
yum -y install docker-ce

touch /etc/docker/daemon.json 

 cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://mirror.aliyuncs.com","https://mirror.baidubce.com", "http://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"],
"log-driver":"json-file",
"log-opts": {"max-file": "20","max-size": "100m"}
}
EOF

systemctl daemon-reload
systemctl restart docker
systemctl enable docker
 
 cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes Repo
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
　　　　https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
enabled=1
EOF


#yum repolist
#yum list all | grep "^kube"
#yum list kubeadm --showduplicates  這個命令可以看到有哪些版本可以安裝
#yum remove kubectl kubeadm kubelet
#安裝自己需要的版本
yum -y install kubectl-1.17.3-0 kubeadm-1.17.3-0 kubelet-1.17.3-0

 cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS="--fail-swap-on=false \
--runtime-cgroups=/systemd/system.slice \
--kubelet-cgroups=/systemd/system.slice \
--allowed-unsafe-sysctls 'kernel.msg*,net.core.somaxconn"
EOF