func cmdAdd(args *skel.CmdArgs) error {
n, err := loadConf(args.StdinData)
if err != nil {
return err
}
netns, err := ns.GetNS(args.Netns)
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", netns, err)
}
defer netns.Close()
if err = createMacvlan(n, args.IfName, netns); err != nil {
return err
}
// run the IPAM plugin and get back the config to apply
result, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return err
}
if result.IP4 == nil {
return errors.New("IPAM plugin returned missing IPv4 config")
}
err = netns.Do(func(_ ns.NetNS) error {
return ipam.ConfigureIface(args.IfName, result)
})
if err != nil {
return err
}
result.DNS = n.DNS
return result.Print()
}
func setupContainerVeth(netns, ifName string, mtu int, pr *types.Result) (string, error) {
// The IPAM result will be something like IP=192.168.3.5/24, GW=192.168.3.1.
// What we want is really a point-to-point link but veth does not support IFF_POINTOPONT.
// Next best thing would be to let it ARP but set interface to 192.168.3.5/32 and
// add a route like "192.168.3.0/24 via 192.168.3.1 dev $ifName".
// Unfortunately that won't work as the GW will be outside the interface's subnet.
// Our solution is to configure the interface with 192.168.3.5/24, then delete the
// "192.168.3.0/24 dev $ifName" route that was automatically added. Then we add
// "192.168.3.1/32 dev $ifName" and "192.168.3.0/24 via 192.168.3.1 dev $ifName".
// In other words we force all traffic to ARP via the gateway except for GW itself.
var hostVethName string
err := ns.WithNetNSPath(netns, func(hostNS ns.NetNS) error {
hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
if err != nil {
return err
}
hostNS.Do(func(_ ns.NetNS) error {
hostVethName = hostVeth.Attrs().Name
if err := ip.SetHWAddrByIP(hostVethName, pr.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
return fmt.Errorf("failed to set hardware addr by IP: %v", err)
}
return nil
})
if err = ipam.ConfigureIface(ifName, pr); err != nil {
return err
}
contVeth, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to look up %q: %v", ifName, err)
}
if err := ip.SetHWAddrByIP(contVeth.Attrs().Name, pr.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
return fmt.Errorf("failed to set hardware addr by IP: %v", err)
}
// Delete the route that was automatically added
route := netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.IP.IP.Mask(pr.IP4.IP.Mask),
Mask: pr.IP4.IP.Mask,
},
Scope: netlink.SCOPE_NOWHERE,
}
if err := netlink.RouteDel(&route); err != nil {
return fmt.Errorf("failed to delete route %v: %v", route, err)
}
for _, r := range []netlink.Route{
netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.Gateway,
Mask: net.CIDRMask(32, 32),
},
Scope: netlink.SCOPE_LINK,
Src: pr.IP4.IP.IP,
},
netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.IP.IP.Mask(pr.IP4.IP.Mask),
Mask: pr.IP4.IP.Mask,
},
Scope: netlink.SCOPE_UNIVERSE,
Gw: pr.IP4.Gateway,
Src: pr.IP4.IP.IP,
},
} {
if err := netlink.RouteAdd(&r); err != nil {
return fmt.Errorf("failed to add route %v: %v", r, err)
}
}
return nil
})
return hostVethName, err
}
// Set up all networking (host/container veth, OVS flows, IPAM, loopback, etc)
func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubehostport.RunningPod, error) {
podConfig, pod, err := m.getPodConfig(req)
if err != nil {
return nil, nil, err
}
ipamResult, err := m.ipamAdd(req.Netns, req.ContainerId)
if err != nil {
// TODO: Remove this hack once we've figured out how to retrieve the netns
// of an exited container. Currently, restarting docker will leak a bunch of
// ips. This will exhaust available ip space unless we cleanup old ips. At the
// same time we don't want to try GC'ing them periodically as that could lead
// to a performance regression in starting pods. So on each setup failure, try
// GC on the assumption that the kubelet is going to retry pod creation, and
// when it does, there will be ips.
m.ipamGarbageCollection()
return nil, nil, fmt.Errorf("failed to run IPAM for %v: %v", req.ContainerId, err)
}
podIP := ipamResult.IP4.IP.IP
// Release any IPAM allocations and hostports if the setup failed
var success bool
defer func() {
if !success {
m.ipamDel(req.ContainerId)
if err := m.hostportHandler.SyncHostports(TUN, m.getRunningPods()); err != nil {
glog.Warningf("failed syncing hostports: %v", err)
}
}
}()
// Open any hostports the pod wants
newPod := &kubehostport.RunningPod{Pod: pod, IP: podIP}
if err := m.hostportHandler.OpenPodHostportsAndSync(newPod, TUN, m.getRunningPods()); err != nil {
return nil, nil, err
}
var hostVeth, contVeth netlink.Link
err = ns.WithNetNSPath(req.Netns, func(hostNS ns.NetNS) error {
hostVeth, contVeth, err = ip.SetupVeth(podInterfaceName, int(m.mtu), hostNS)
if err != nil {
return fmt.Errorf("failed to create container veth: %v", err)
}
// refetch to get hardware address and other properties
contVeth, err = netlink.LinkByIndex(contVeth.Attrs().Index)
if err != nil {
return fmt.Errorf("failed to fetch container veth: %v", err)
}
// Clear out gateway to prevent ConfigureIface from adding the cluster
// subnet via the gateway
ipamResult.IP4.Gateway = nil
if err = ipam.ConfigureIface(podInterfaceName, ipamResult); err != nil {
return fmt.Errorf("failed to configure container IPAM: %v", err)
}
lo, err := netlink.LinkByName("lo")
if err == nil {
err = netlink.LinkSetUp(lo)
}
if err != nil {
return fmt.Errorf("failed to configure container loopback: %v", err)
}
return nil
})
if err != nil {
return nil, nil, err
}
if podConfig.wantMacvlan {
if err := addMacvlan(req.Netns); err != nil {
return nil, nil, err
}
}
contVethMac := contVeth.Attrs().HardwareAddr.String()
vnidStr := vnidToString(podConfig.vnid)
out, err := exec.Command(sdnScript, setUpCmd, hostVeth.Attrs().Name, contVethMac, podIP.String(), vnidStr, podConfig.ingressBandwidth, podConfig.egressBandwidth).CombinedOutput()
glog.V(5).Infof("SetUpPod network plugin output: %s, %v", string(out), err)
if isScriptError(err) {
return nil, nil, fmt.Errorf("error running network setup script:\nhostVethName %s, contVethMac %s, podIP %s, podConfig %#v\n %s", hostVeth.Attrs().Name, contVethMac, podIP.String(), podConfig, getScriptError(out))
} else if err != nil {
return nil, nil, err
}
success = true
return ipamResult, newPod, nil
}
func cmdAdd(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
br, err := setupBridge(n)
if err != nil {
return err
}
if err = setupVeth(args.Netns, br, args.IfName, n.MTU, n.HairpinMode); err != nil {
return err
}
// run the IPAM plugin and get back the config to apply
result, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return err
}
if result.IP4 == nil {
return errors.New("IPAM plugin returned missing IPv4 config")
}
if result.IP4.Gateway == nil && n.IsGW {
result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
}
err = ns.WithNetNSPath(args.Netns, false, func(hostNS *os.File) error {
return ipam.ConfigureIface(args.IfName, result)
})
if err != nil {
return err
}
if n.IsGW {
gwn := &net.IPNet{
IP: result.IP4.Gateway,
Mask: result.IP4.IP.Mask,
}
if err = ensureBridgeAddr(br, gwn); err != nil {
return err
}
if err := ip.EnableIP4Forward(); err != nil {
return fmt.Errorf("failed to enable forwarding: %v", err)
}
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
return err
}
}
result.DNS = n.DNS
return result.Print()
}
// Set up all networking (host/container veth, OVS flows, IPAM, loopback, etc)
func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubehostport.RunningPod, error) {
podConfig, pod, err := m.getPodConfig(req)
if err != nil {
return nil, nil, err
}
ipamResult, err := m.runIPAM(req.Netns, cniserver.CNI_ADD, req.ContainerId)
if err != nil {
return nil, nil, fmt.Errorf("failed to run IPAM for %v: %v", req.ContainerId, err)
}
podIP := ipamResult.IP4.IP.IP
// Release any IPAM allocations and hostports if the setup failed
var success bool
defer func() {
if !success {
m.runIPAM(req.Netns, cniserver.CNI_DEL, req.ContainerId)
if err := m.hostportHandler.SyncHostports(TUN, m.getRunningPods()); err != nil {
glog.Warningf("failed syncing hostports: %v", err)
}
}
}()
// Open any hostports the pod wants
newPod := &kubehostport.RunningPod{Pod: pod, IP: podIP}
if err := m.hostportHandler.OpenPodHostportsAndSync(newPod, TUN, m.getRunningPods()); err != nil {
return nil, nil, err
}
var hostVeth, contVeth netlink.Link
err = ns.WithNetNSPath(req.Netns, func(hostNS ns.NetNS) error {
hostVeth, contVeth, err = ip.SetupVeth(podInterfaceName, int(m.mtu), hostNS)
if err != nil {
return fmt.Errorf("failed to create container veth: %v", err)
}
// refetch to get hardware address and other properties
contVeth, err = netlink.LinkByIndex(contVeth.Attrs().Index)
if err != nil {
return fmt.Errorf("failed to fetch container veth: %v", err)
}
// Clear out gateway to prevent ConfigureIface from adding the cluster
// subnet via the gateway
ipamResult.IP4.Gateway = nil
if err = ipam.ConfigureIface(podInterfaceName, ipamResult); err != nil {
return fmt.Errorf("failed to configure container IPAM: %v", err)
}
lo, err := netlink.LinkByName("lo")
if err == nil {
err = netlink.LinkSetUp(lo)
}
if err != nil {
return fmt.Errorf("failed to configure container loopback: %v", err)
}
return nil
})
if err != nil {
return nil, nil, err
}
if podConfig.wantMacvlan {
if err := addMacvlan(req.Netns); err != nil {
return nil, nil, err
}
}
contVethMac := contVeth.Attrs().HardwareAddr.String()
vnidStr := vnidToString(podConfig.vnid)
out, err := exec.Command(sdnScript, setUpCmd, hostVeth.Attrs().Name, contVethMac, podIP.String(), vnidStr, podConfig.ingressBandwidth, podConfig.egressBandwidth).CombinedOutput()
glog.V(5).Infof("SetUpPod network plugin output: %s, %v", string(out), err)
if isScriptError(err) {
return nil, nil, fmt.Errorf("error running network setup script:\nhostVethName %s, contVethMac %s, podIP %s, podConfig %#v\n %s", hostVeth.Attrs().Name, contVethMac, podIP.String(), podConfig, getScriptError(out))
} else if err != nil {
return nil, nil, err
}
success = true
return ipamResult, newPod, nil
}
func cmdAdd(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
if n.IsDefaultGW {
n.IsGW = true
}
br, err := setupBridge(n)
if err != nil {
return err
}
netns, err := ns.GetNS(args.Netns)
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
}
defer netns.Close()
if err = setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode); err != nil {
return err
}
// run the IPAM plugin and get back the config to apply
result, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return err
}
// TODO: make this optional when IPv6 is supported
if result.IP4 == nil {
return errors.New("IPAM plugin returned missing IPv4 config")
}
if result.IP4.Gateway == nil && n.IsGW {
result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
}
if err := netns.Do(func(_ ns.NetNS) error {
// set the default gateway if requested
if n.IsDefaultGW {
_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
if err != nil {
return err
}
for _, route := range result.IP4.Routes {
if defaultNet.String() == route.Dst.String() {
if route.GW != nil && !route.GW.Equal(result.IP4.Gateway) {
return fmt.Errorf(
"isDefaultGateway ineffective because IPAM sets default route via %q",
route.GW,
)
}
}
}
result.IP4.Routes = append(
result.IP4.Routes,
types.Route{Dst: *defaultNet, GW: result.IP4.Gateway},
)
// TODO: IPV6
}
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
return err
}
if err := ip.SetHWAddrByIP(args.IfName, result.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
return err
}
return nil
}); err != nil {
return err
}
if n.IsGW {
gwn := &net.IPNet{
IP: result.IP4.Gateway,
Mask: result.IP4.IP.Mask,
}
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
return err
}
if err := ip.SetHWAddrByIP(n.BrName, gwn.IP, nil /* TODO IPv6 */); err != nil {
return err
}
if err := ip.EnableIP4Forward(); err != nil {
return fmt.Errorf("failed to enable forwarding: %v", err)
}
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
return err
}
}
result.DNS = n.DNS
return result.Print()
}