k8s設計了網路模型，但卻將他的實現交給了網路外掛，CNI網路外掛最主要的功能
就是實現POD資源能夠跨主機進行通訊

最常見的CNI網路外掛：
Flannel、Calico、Cancl、OpenContrail、NSX-T、Kube-router

flannel下載地址
https://github.com/coreos/flannel/releases
wget https://github.com/coreos/flannel/releases/download/v0.12.0/flannel-v0.12.0-linux-amd64.tar.gz

長傳安裝包
[root@rstx-203 ~]# cd /opt/src/
[root@rstx-203 src]# ls
flannel-v0.12.0-linux-amd64.tar.gz

[root@rstx-203 src]# mkdir /opt/flannel-v0.12.0

[root@rstx-203 src]# tar xf flannel-v0.12.0-linux-amd64.tar.gz -C /opt/flannel-v0.12.0/

建立軟連結
[root@rstx-203 src]# ln -s /opt/flannel-v0.12.0/ /opt/flannel

[root@rstx-203 src]# cd ..
[root@rstx-203 opt]# ll
總用量 0
drwx--x--x 4 root root 28 12月 17 10:52 containerd
lrwxrwxrwx 1 root root 21 12月 18 09:48 flannel -> /opt/flannel-v0.12.0/
drwxr-xr-x 2 root root 64 12月 18 09:47 flannel-v0.12.0
lrwxrwxrwx 1 root root 24 12月 17 10:35 kubernetes -> /opt/kubernetes-v1.15.2/
drwxr-xr-x 4 root root 50 12月 17 10:35 kubernetes-v1.15.2
drwxr-xr-x 2 root root 100 12月 18 09:46 src

拷貝證書
[root@rstx-203 opt]# cd flannel
[root@rstx-203 flannel]# mkdir certs
[root@rstx-203 flannel]# cd certs/

[root@rstx-203 certs]# scp hdss7-248:/opt/certs/ca.pem .
[root@rstx-203 certs]# scp hdss7-248:/opt/certs/client.pem .
[root@rstx-203 certs]# scp hdss7-248:/opt/certs/client-key.pem .

[root@rstx-203 certs]# cd ..

[root@rstx-203 flannel]# vi subnet.env
FLANNEL_NETWORK=172.7.0.0/16 # pod的網段
FLANNEL_SUBNET=172.7.21.1/24 # 本機執行pod的網段
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false

建立啟動指令碼
[root@rstx-203 flannel]# vi flanneld.sh

!/bin/sh

./flanneld
--public-ip=192.168.1.203 \ # 本機IP
--etcd-endpoints=https://192.168.1.202:2379,https://192.168.1.203:2379,https://192.168.1.204:2379 \ # etcd的IP
--etcd-keyfile=./certs/client-key.pem
--etcd-certsfile=./certs/client.pem
--etcd-cafile=./certs/ca.pem
--iface=enp2s0 \ # 注意更改網絡卡名稱
--subnet-file=./subnet.env
--healthz-port=2401

[root@rstx-203 flannel]# chmod +x flanneld.sh

[root@rstx-203 flannel]# mkdir -p /data/logs/flanneld

操作etcd，增加host-gw模型
[root@rstx-203 ~]# cd /opt/etcd

Flannel的host-gw模型，所有node ip必須在同一個物理閘道器裝置下才能使用
[root@rstx-203etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}

檢視etcd叢集
[root@rstx-203 etcd]# ./etcdctl member list
1221562a073d0c1a: name=etcd-server-203 peerURLs=https://192.168.1.203:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.203:2379 isLeader=true
89fe55dc5058e687: name=etcd-server-204 peerURLs=https://192.168.1.204:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.204:2379 isLeader=false
af26e2bc543fbfba: name=etcd-server-202 peerURLs=https://192.168.1.202:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.202:2379 isLeader=false

檢視flanneld網路
[root@rstx-203 etcd]# ./etcdctl get /coreos.com/network/config
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}

[root@rstx-203 flannel]# vi /etc/supervisord.d/flannel.ini
[program:flanneld-203]
command=/opt/flannel/flanneld.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/flannel ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=30 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=true ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
killasgroup=true
stopasgroup=true

[root@rstx-203 flannel]# supervisorctl update
flanneld-7-121: added process group
[root@rstx-203 flannel]# supervisorctl status
flanneld-7-121 RUNNING pid 9463, uptime 0:00:54
kube-kubelet-7-121 RUNNING pid 6683, uptime 18:24:51
kube-proxy-7-121 RUNNING pid 32574, uptime 17:50:07

詳解flanneld工作原理
[root@rstx-203 etcd]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.1.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.7.67.0 192.168.1.204 255.255.255.0 UG 0 0 0 eth0
172.7.203.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0

flannel原理就是：給宿主機新增一個靜態路由，到達pod ip

Flannel的host-gw模型，所有node ip必須在同一個物理網管裝置下才能使用

flannel的其他網路模型

Vxlan模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'
[root@hdss7-14 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'

直接路由模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN","Directrouting": true}}'

更改flanneld的網路模型
[root@rstx-203 flannel]# supervisorctl stop flanneld-7-11
[root@rstx-203 flannel]# ps aux | grep flanneld
[root@rstx-203 flannel]# kill
刪除路由
[root@rstx-203 flannel]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.16.1 0.0.0.0 UG 100 0 0 enp2s0
172.7.11.0 192.168.16.11 255.255.255.0 UG 0 0 0 enp2s0
172.7.12.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
172.7.13.0 192.168.16.13 255.255.255.0 UG 0 0 0 enp2s0
172.7.14.0 192.168.16.14 255.255.255.0 UG 0 0 0 enp2s0
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 br-7957b5310092
192.168.16.0 0.0.0.0 255.255.255.0 U 100 0 0 enp2s0

[root@rstx-203 flannel]# route del -net 172.7.12.0/24 gw 192.168.16.12
[root@rstx-203 flannel]# route del -net 172.7.11.0/24 gw 192.168.16.11

[root@rstx-203 flannel]# ./etcdctl rm /coreos.com/network/config

[root@rstx-203 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'

[root@rstx-203 flannel]# supervisorctl start flanneld-7-11

直接路由模型 -- 當node不在同一個物理閘道器下，走vxaln模型,在同一個閘道器下，走host-gw模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN","Directrouting": true}}'

iptables優化

所有節點執行 -- 不優化的話，iptables會佔有大量的資源

~]# yum install iptables-services -y

~]# systemctl start iptables

~]# systemctl enable iptables

~]# iptables-save | grep -i postrouting

此條命令。重啟docker會自動新增上，需要寫個指令碼，每次開機自動自動執行

~]# iptables -t nat -D POSTROUTING -s 172.7.203.0/24 ! -o docker0 -j MASQUERADE # 刪除172.7.203.0/24 為此主機上執行的pod網段

~]# iptables -t nat -I POSTROUTING -s 172.7.203.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE # 新增172.7.203.0/24 為此主機上執行的pod網段

~]# iptables-save |grep -i postrouting

~]# iptables-save > /etc/sysconfig/iptables

192.168.1.203主機上，來源是172.7.203.0/24段的docker的ip，目標ip不是172.7.0.0/16段，網路發包不從docker0橋裝置上出站的，才進行SNAT轉換，pod之間看到的是真實的pod ip

刪除 filter表上的兩個規則

[root@rstx-203 ~]# iptables-save | grep -i reject

[root@rstx-203 ~]# iptables -t filter -D INPUT -j REJECT --reject-with icmp-host-prohibited
[root@rstx-203 ~]# iptables -t filter -D FORWARD -j REJECT --reject-with icmp-host-prohibited

~]# iptables-save > /etc/sysconfig/iptables

[root@rstx-203 ~]# service iptables save