容器CNI完全解讀bridge實現(二)
之前介紹CNI基本操作,現在介紹一個bridge的實現。
它也實現了建立和刪除介面。
先看建立介面:
func cmdAdd(args *skel.CmdArgs) error {
n, cniVersion, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
if n.IsDefaultGW {
n.IsGW = true
}
br, brInterface, err := setupBridge(n)
if err != nil {
return err
}
netns, err := ns.GetNS (args.Netns)
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
}
defer netns.Close()
hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode)
if err != nil {
return err
}
// run the IPAM plugin and get back the config to apply
r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return err
}
// Convert whatever the IPAM result was into the current Result type
result, err := current.NewResultFromResult(r)
if err != nil {
return err
}
if len(result.IPs ) == 0 {
return errors.New("IPAM plugin returned missing IP config")
}
result.Interfaces = []*current.Interface{brInterface, hostInterface, containerInterface}
for _, ipc := range result.IPs {
// All IPs currently refer to the container interface
ipc.Interface = 2
if ipc.Gateway == nil && n.IsGW {
ipc.Gateway = calcGatewayIP(&ipc.Address)
}
}
if err := netns.Do(func(_ ns.NetNS) error {
// set the default gateway if requested
if n.IsDefaultGW {
for _, ipc := range result.IPs {
defaultNet := &net.IPNet{}
switch {
case ipc.Address.IP.To4() != nil:
defaultNet.IP = net.IPv4zero
defaultNet.Mask = net.IPMask(net.IPv4zero)
case len(ipc.Address.IP) == net.IPv6len && ipc.Address.IP.To4() == nil:
defaultNet.IP = net.IPv6zero
defaultNet.Mask = net.IPMask(net.IPv6zero)
default:
return fmt.Errorf("Unknown IP object: %v", ipc)
}
for _, route := range result.Routes {
if defaultNet.String() == route.Dst.String() {
if route.GW != nil && !route.GW.Equal(ipc.Gateway) {
return fmt.Errorf(
"isDefaultGateway ineffective because IPAM sets default route via %q",
route.GW,
)
}
}
}
result.Routes = append(
result.Routes,
&types.Route{Dst: *defaultNet, GW: ipc.Gateway},
)
}
}
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
return err
}
if err := ip.SetHWAddrByIP(args.IfName, result.IPs[0].Address.IP, nil /* TODO IPv6 */); err != nil {
return err
}
// Refetch the veth since its MAC address may changed
link, err := netlink.LinkByName(args.IfName)
if err != nil {
return fmt.Errorf("could not lookup %q: %v", args.IfName, err)
}
containerInterface.Mac = link.Attrs().HardwareAddr.String()
return nil
}); err != nil {
return err
}
if n.IsGW {
var firstV4Addr net.IP
for _, ipc := range result.IPs {
gwn := &net.IPNet{
IP: ipc.Gateway,
Mask: ipc.Address.Mask,
}
if ipc.Gateway.To4() != nil && firstV4Addr == nil {
firstV4Addr = ipc.Gateway
}
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
return err
}
}
if firstV4Addr != nil {
if err := ip.SetHWAddrByIP(n.BrName, firstV4Addr, nil /* TODO IPv6 */); err != nil {
return err
}
}
if err := ip.EnableIP4Forward(); err != nil {
return fmt.Errorf("failed to enable forwarding: %v", err)
}
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
for _, ipc := range result.IPs {
if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil {
return err
}
}
}
// Refetch the bridge since its MAC address may change when the first
// veth is added or after its IP address is set
br, err = bridgeByName(n.BrName)
if err != nil {
return err
}
brInterface.Mac = br.Attrs().HardwareAddr.String()
result.DNS = n.DNS
return types.PrintResult(result, cniVersion)
}
這個裡面有幾個步驟
1、建立網橋並啟動網橋,
setupBridge裡面呼叫ensureBridge,先通過err := netlink.LinkAdd(br)建立網橋,然後通過 err := netlink.LinkSetUp(br)啟動網橋
2.建立veth,這個是一個管道,Linux的網絡卡對。
hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode)
當前先通過hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)建立網絡卡對,一個是主機網絡卡一個容器網絡卡,並且把主機網絡卡設定成hairpin模式
3.通過ipam獲取IP,這個是呼叫另一個獲取IP的二進位制檔案ipam,這個裡面返回一個網路配置列表(result.IPs)
4.配置容器內網絡卡,通過ipam.ConfigureIface和ip.SetHWAddrByIP配置容器的IP地址和網絡卡mac,其中SetHWAddrByIP通過IP地址計算出mac地址,詳見
hwAddr, err := hwaddr.GenerateHardwareAddr4(ip4, hwaddr.PrivateMACPrefix)
5.配置網橋,這個裡面配置網橋的IP地址和mac。最後如果可以設定ipmasq,這樣容器就可以通過SNAT去連線外部網路了,這樣容器就可以加入網路了
介紹完建立的過程,刪除的過程就簡單了。
func cmdDel(args *skel.CmdArgs) error {
n, _, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
return err
}
if args.Netns == "" {
return nil
}
var ipn *net.IPNet
err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error {
var err error
ipn, err = ip.DelLinkByNameAddr(args.IfName, netlink.FAMILY_V4)
return err
})
if err != nil {
return err
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.TeardownIPMasq(ipn, chain, comment); err != nil {
return err
}
}
return nil
}
這個裡面顯示通過ipam.ExecDel是釋放IP地址的佔用,然後通過 ip.DelLinkByNameAddr去刪除veth,如果設定了ipmasq,這是就可以通過TeardownIPMasq刪除這些iptables規則。