k8s cni

作者: 分享放大价值 | 来源:发表于2020-06-07 15:05 被阅读0次

k8s cni
K8S 网络插件对比
k8s之ovs-cni
k8s flannel host-gw
查看CNI中的veth pair
k8s 网络三
压测iptables规则条数的限制
k8s Flannel网络方案
k8s cni
K8S CNI

https://github.com/containernetworking/cni/blob/master/pkg/skel/skel.go
skel为cni提供了统一的框架，解析下面六个环境变量和标准输入内容(每个cni插件的配置文件)

CNI_COMMAND   -- 只支持这四个命令: ADD,DEL,CHECK,VERSION
CNI_CONTAINERID
CNI_NETNS
CNI_IFNAME
CNI_ARGS
CNI_PATH

然后创建cmdArgs

    cmdArgs := &CmdArgs{
        ContainerID: contID,   //对应CNI_CONTAINERID
        Netns:       netns,       //对应CNI_NETNS
        IfName:      ifName,   //对应CNI_IFNAME
        Args:        args,         //对应CNI_ARGS
        Path:        path,         //对应CNI_PATH, CNI插件所在目录
        StdinData:   stdinData, //一般是CNI插件的配置内容
    }
    return cmd, cmdArgs, nil

比如host-device插件，调用PluginMain时，提供了四个命令分别对应的函数和版本信息cmdAdd, cmdCheck, cmdDel, version.All，如果skel解析环境变量和校验成功后，根据CNI_COMMAND指定的cmd调用相应的函数

func main() {
    skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("host-device"))
}

host-device cni

host-device是cni官方提供的插件，实现比较简单，可以用来学习cni。
作用是把host上的网卡设备移动到容器里。
原理也比较简单，使用 "ip link set dev NAME netns x"命令就可以做到。

#将host上的ens8移动到netns test中
root@node1:~# ip netns add test
root@node1:~# ip netns
test (id: 0)
root@node1:~# ip a
...
3: ens8: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 52:54:00:5e:1f:f5 brd ff:ff:ff:ff:ff:ff
...
#移动ens8到netns，此时在host上看不到ens8了
root@node1:~# ip link set dev ens8 netns test
#在netns中查看，确实有ens8
root@node1:~# ip netns exec test ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
3: ens8: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 52:54:00:5e:1f:f5 brd ff:ff:ff:ff:ff:ff
#将ens8返回到host上，1代表root netns，即host的netns
root@node1:~# ip netns exec test ip link set ens8 netns 1
root@node1:~# ip netns exec test ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
root@node1:~# ip a
3: ens8: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 52:54:00:5e:1f:f5 brd ff:ff:ff:ff:ff:ff

每个插件都有它自己的配置信息

//NetConf for host-device config, look the README to learn how to use those parameters
type NetConf struct {
    types.NetConf
    Device        string `json:"device"`     // Device-Name, something like eth0 or can0 etc.
    HWAddr        string `json:"hwaddr"`     // MAC Address of target network interface
    KernelPath    string `json:"kernelpath"` // Kernelpath of the device
    PCIAddr       string `json:"pciBusID"`   // PCI Address of target network device
    RuntimeConfig struct {
        DeviceID string `json:"deviceID,omitempty"`
    } `json:"runtimeConfig,omitempty"`
}
func loadConf(bytes []byte) (*NetConf, error) {
    n := &NetConf{}
    if err := json.Unmarshal(bytes, n); err != nil {
        return nil, fmt.Errorf("failed to load netconf: %v", err)
    }

    if n.RuntimeConfig.DeviceID != "" {
        // Override PCI device with the standardized DeviceID provided in Runtime Config.
        n.PCIAddr = n.RuntimeConfig.DeviceID
    }

    if n.Device == "" && n.HWAddr == "" && n.KernelPath == "" && n.PCIAddr == "" {
        return nil, fmt.Errorf(`specify either "device", "hwaddr", "kernelpath" or "pciBusID"`)
    }

    return n, nil
}

下面开始尝试cni提供的这几个命令

VERSION cmd

$ export CNI_COMMAND=VERSION
$ /opt/cni/bin/host-device
{"cniVersion":"0.4.0","supportedVersions":["0.1.0","0.2.0","0.3.0","0.3.1","0.4.0"]}

ADD cmd

通过host-device cni插件实现将host上的ens8移动到netns test中

root@node1:~# export CNI_COMMAND=ADD
root@node1:~# export CNI_NETNS=/var/run/netns/test
root@node1:~# export CNI_IFNAME=eth0
root@node1:~# export CNI_ARGS
root@node1:~# export CNI_PATH=/opt/cni/bin
root@node1:~# export CNI_CONTAINERID="aaa" 
root@node1:~#
root@node1:~#  /opt/cni/bin/host-device <<EOF
> {
> "cniVersion": "0.3.1",
> "type": "host-device",
> "device": "ens8",
> "name": "host"
> }
> EOF
{
    "cniVersion": "0.3.1",
    "interfaces": [
        {
            "name": "eth0",
            "mac": "52:54:00:5e:1f:f5",
            "sandbox": "/var/run/netns/test"
        }
    ],
    "dns": {}
}root@node1:~# ip netns exec test ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
3: eth0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 52:54:00:5e:1f:f5 brd ff:ff:ff:ff:ff:ff

DEL cmd

root@node1:~# export CNI_COMMAND=DEL
root@node1:~# export CNI_NETNS=/var/run/netns/test
root@node1:~# export CNI_IFNAME=eth0
root@node1:~# export CNI_ARGS
root@node1:~# export CNI_PATH=/opt/cni/bin
root@node1:~#
root@node1:~#  /opt/cni/bin/host-device <<EOF
> {
> "cniVersion": "0.3.1",
> "type": "host-device",
> "device": "ens8",
> "name": "host"
> }
> EOF
root@node1:~# ip netns exec test ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0