// teardownKvmNets teardown every active networking from networking by
// removing tuntap interface and releasing its ip from IPAM plugin
func (n *Networking) teardownKvmNets() {
for _, an := range n.nets {
if an.conf.Type == "flannel" {
if err := kvmTransformFlannelNetwork(&an); err != nil {
stderr.PrintE("error transforming flannel network", err)
continue
}
}
switch an.conf.Type {
case "ptp", "bridge":
// remove tuntap interface
tuntap.RemovePersistentIface(an.runtime.IfName, tuntap.Tap)
case "macvlan":
link, err := netlink.LinkByName(an.runtime.IfName)
if err != nil {
stderr.PrintE(fmt.Sprintf("cannot find link `%v`", an.runtime.IfName), err)
continue
} else {
err := netlink.LinkDel(link)
if err != nil {
stderr.PrintE(fmt.Sprintf("cannot remove link `%v`", an.runtime.IfName), err)
continue
}
}
default:
stderr.Printf("unsupported network type: %q", an.conf.Type)
continue
}
// ugly hack again to directly call IPAM plugin to release IP
an.conf.Type = an.conf.IPAM.Type
_, err := n.execNetPlugin("DEL", &an, an.runtime.IfName)
if err != nil {
stderr.PrintE("error executing network plugin", err)
}
// remove masquerading if it was prepared
if an.conf.IPMasq {
chain := cniutils.FormatChainName(an.conf.Name, n.podID.String())
comment := cniutils.FormatChainName(an.conf.Name, n.podID.String())
err := ip.TeardownIPMasq(&net.IPNet{
IP: an.runtime.IP,
Mask: net.IPMask(an.runtime.Mask),
}, chain, comment)
if err != nil {
stderr.PrintE("error on removing masquerading", err)
}
}
}
}
func cmdDel(args *skel.CmdArgs) error {
conf := NetConf{}
if err := json.Unmarshal(args.StdinData, &conf); err != nil {
return fmt.Errorf("failed to load netconf: %v", err)
}
if err := ipam.ExecDel(conf.IPAM.Type, args.StdinData); err != nil {
return err
}
if args.Netns == "" {
return nil
}
var ipn *net.IPNet
err := ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error {
var err error
ipn, err = ip.DelLinkByNameAddr(args.IfName, netlink.FAMILY_V4)
return err
})
if err != nil {
return err
}
if conf.IPMasq {
chain := utils.FormatChainName(conf.Name, args.ContainerID)
comment := utils.FormatComment(conf.Name, args.ContainerID)
if err = ip.TeardownIPMasq(ipn, chain, comment); err != nil {
return err
}
}
return nil
}
func cmdDel(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
return err
}
var ipn *net.IPNet
err = ns.WithNetNSPath(args.Netns, false, func(hostNS *os.File) error {
var err error
ipn, err = ip.DelLinkByNameAddr(args.IfName, netlink.FAMILY_V4)
return err
})
if err != nil {
return err
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.TeardownIPMasq(ipn, chain, comment); err != nil {
return err
}
}
return nil
}
func cmdAdd(args *skel.CmdArgs) error {
conf := NetConf{}
if err := json.Unmarshal(args.StdinData, &conf); err != nil {
return fmt.Errorf("failed to load netconf: %v", err)
}
if err := ip.EnableIP4Forward(); err != nil {
return fmt.Errorf("failed to enable forwarding: %v", err)
}
// run the IPAM plugin and get back the config to apply
result, err := ipam.ExecAdd(conf.IPAM.Type, args.StdinData)
if err != nil {
return err
}
if result.IP4 == nil {
return errors.New("IPAM plugin returned missing IPv4 config")
}
hostVethName, err := setupContainerVeth(args.Netns, args.IfName, conf.MTU, result)
if err != nil {
return err
}
if err = setupHostVeth(hostVethName, result.IP4); err != nil {
return err
}
if conf.IPMasq {
chain := utils.FormatChainName(conf.Name, args.ContainerID)
comment := utils.FormatComment(conf.Name, args.ContainerID)
if err = ip.SetupIPMasq(&result.IP4.IP, chain, comment); err != nil {
return err
}
}
result.DNS = conf.DNS
return result.Print()
}
func cmdAdd(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
br, err := setupBridge(n)
if err != nil {
return err
}
if err = setupVeth(args.Netns, br, args.IfName, n.MTU, n.HairpinMode); err != nil {
return err
}
// run the IPAM plugin and get back the config to apply
result, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return err
}
if result.IP4 == nil {
return errors.New("IPAM plugin returned missing IPv4 config")
}
if result.IP4.Gateway == nil && n.IsGW {
result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
}
err = ns.WithNetNSPath(args.Netns, false, func(hostNS *os.File) error {
return ipam.ConfigureIface(args.IfName, result)
})
if err != nil {
return err
}
if n.IsGW {
gwn := &net.IPNet{
IP: result.IP4.Gateway,
Mask: result.IP4.IP.Mask,
}
if err = ensureBridgeAddr(br, gwn); err != nil {
return err
}
if err := ip.EnableIP4Forward(); err != nil {
return fmt.Errorf("failed to enable forwarding: %v", err)
}
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
return err
}
}
result.DNS = n.DNS
return result.Print()
}
func cmdAdd(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
if n.IsDefaultGW {
n.IsGW = true
}
br, err := setupBridge(n)
if err != nil {
return err
}
netns, err := ns.GetNS(args.Netns)
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
}
defer netns.Close()
if err = setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode); err != nil {
return err
}
// run the IPAM plugin and get back the config to apply
result, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return err
}
// TODO: make this optional when IPv6 is supported
if result.IP4 == nil {
return errors.New("IPAM plugin returned missing IPv4 config")
}
if result.IP4.Gateway == nil && n.IsGW {
result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
}
if err := netns.Do(func(_ ns.NetNS) error {
// set the default gateway if requested
if n.IsDefaultGW {
_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
if err != nil {
return err
}
for _, route := range result.IP4.Routes {
if defaultNet.String() == route.Dst.String() {
if route.GW != nil && !route.GW.Equal(result.IP4.Gateway) {
return fmt.Errorf(
"isDefaultGateway ineffective because IPAM sets default route via %q",
route.GW,
)
}
}
}
result.IP4.Routes = append(
result.IP4.Routes,
types.Route{Dst: *defaultNet, GW: result.IP4.Gateway},
)
// TODO: IPV6
}
if err := ipam.ConfigureIface(args.IfName, result); err != nil {
return err
}
if err := ip.SetHWAddrByIP(args.IfName, result.IP4.IP.IP, nil /* TODO IPv6 */); err != nil {
return err
}
return nil
}); err != nil {
return err
}
if n.IsGW {
gwn := &net.IPNet{
IP: result.IP4.Gateway,
Mask: result.IP4.IP.Mask,
}
if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
return err
}
if err := ip.SetHWAddrByIP(n.BrName, gwn.IP, nil /* TODO IPv6 */); err != nil {
return err
}
if err := ip.EnableIP4Forward(); err != nil {
return fmt.Errorf("failed to enable forwarding: %v", err)
}
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
return err
}
}
result.DNS = n.DNS
return result.Print()
}
// kvmSetup prepare new Networking to be used in kvm environment based on tuntap pair interfaces
// to allow communication with virtual machine created by lkvm tool
func kvmSetup(podRoot string, podID types.UUID, fps []ForwardedPort, netList common.NetList, localConfig string, noDNS bool) (*Networking, error) {
network := Networking{
podEnv: podEnv{
podRoot: podRoot,
podID: podID,
netsLoadList: netList,
localConfig: localConfig,
},
}
var e error
// If there's a network set as default in CNI configuration
defaultGatewaySet := false
_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
if err != nil {
return nil, errwrap.Wrap(errors.New("error when parsing net address"), err)
}
network.nets, e = network.loadNets()
if e != nil {
return nil, errwrap.Wrap(errors.New("error loading network definitions"), e)
}
// did stage0 already make /etc/rkt-resolv.conf (i.e. --dns passed)
resolvPath := filepath.Join(common.Stage1RootfsPath(podRoot), "etc/rkt-resolv.conf")
_, err = os.Stat(resolvPath)
if err != nil && !os.IsNotExist(err) {
return nil, errwrap.Wrap(fmt.Errorf("error statting /etc/rkt-resolv.conf"), err)
}
podHasResolvConf := err == nil
for i, n := range network.nets {
if n.conf.Type == "flannel" {
if err := kvmTransformFlannelNetwork(&n); err != nil {
return nil, errwrap.Wrap(errors.New("cannot transform flannel network into basic network"), err)
}
}
switch n.conf.Type {
case "ptp":
link, err := setupTapDevice(podID)
if err != nil {
return nil, err
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
// add address to host tap device
err = ensureHasAddr(
link,
&net.IPNet{
IP: n.runtime.IP4.Gateway,
Mask: net.IPMask(n.runtime.Mask),
},
)
if err != nil {
return nil, errwrap.Wrap(fmt.Errorf("cannot add address to host tap device %q", ifName), err)
}
if err := removeAllRoutesOnLink(link); err != nil {
return nil, errwrap.Wrap(fmt.Errorf("cannot remove route on host tap device %q", ifName), err)
}
if err := addRoute(link, n.runtime.IP); err != nil {
return nil, errwrap.Wrap(errors.New("cannot add on host direct route to pod"), err)
}
case "bridge":
config := BridgeNetConf{
NetConf: NetConf{
MTU: defaultMTU,
},
BrName: defaultBrName,
}
if err := json.Unmarshal(n.confBytes, &config); err != nil {
return nil, errwrap.Wrap(fmt.Errorf("error parsing %q result", n.conf.Name), err)
}
br, err := ensureBridgeIsUp(config.BrName, config.MTU)
if err != nil {
return nil, errwrap.Wrap(errors.New("error in time of bridge setup"), err)
}
link, err := setupTapDevice(podID)
if err != nil {
return nil, errwrap.Wrap(errors.New("can not setup tap device"), err)
}
err = netlink.LinkSetMaster(link, br)
if err != nil {
rErr := tuntap.RemovePersistentIface(n.runtime.IfName, tuntap.Tap)
if rErr != nil {
stderr.PrintE("warning: could not cleanup tap interface", rErr)
}
return nil, errwrap.Wrap(errors.New("can not add tap interface to bridge"), err)
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
if n.conf.IsDefaultGateway {
n.runtime.IP4.Routes = append(
n.runtime.IP4.Routes,
cnitypes.Route{Dst: *defaultNet, GW: n.runtime.IP4.Gateway},
)
defaultGatewaySet = true
config.IsGw = true
}
if config.IsGw {
err = ensureHasAddr(
br,
&net.IPNet{
IP: n.runtime.IP4.Gateway,
Mask: net.IPMask(n.runtime.Mask),
},
)
if err != nil {
return nil, errwrap.Wrap(fmt.Errorf("cannot add address to host bridge device %q", br.Name), err)
}
}
case "macvlan":
config := MacVTapNetConf{}
if err := json.Unmarshal(n.confBytes, &config); err != nil {
return nil, errwrap.Wrap(fmt.Errorf("error parsing %q result", n.conf.Name), err)
}
link, err := setupMacVTapDevice(podID, config, i)
if err != nil {
return nil, err
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("network %q have unsupported type: %q", n.conf.Name, n.conf.Type)
}
// Check if there's any other network set as default gateway
if defaultGatewaySet {
for _, route := range n.runtime.IP4.Routes {
if (defaultNet.String() == route.Dst.String()) && !n.conf.IsDefaultGateway {
return nil, fmt.Errorf("flannel config enables default gateway and IPAM sets default gateway via %q", n.runtime.IP4.Gateway)
}
}
}
// Generate rkt-resolv.conf if it's not already there.
// The first network plugin that supplies a non-empty
// DNS response will win, unless noDNS is true (--dns passed to rkt run)
if !common.IsDNSZero(&n.runtime.DNS) && !noDNS {
if !podHasResolvConf {
err := ioutil.WriteFile(
resolvPath,
[]byte(common.MakeResolvConf(n.runtime.DNS, "Generated by rkt from network "+n.conf.Name)),
0644)
if err != nil {
return nil, errwrap.Wrap(fmt.Errorf("error creating resolv.conf"), err)
}
podHasResolvConf = true
} else {
stderr.Printf("Warning: network %v plugin specified DNS configuration, but DNS already supplied", n.conf.Name)
}
}
if n.conf.IPMasq {
chain := cniutils.FormatChainName(n.conf.Name, podID.String())
comment := cniutils.FormatComment(n.conf.Name, podID.String())
if err := ip.SetupIPMasq(&net.IPNet{
IP: n.runtime.IP,
Mask: net.IPMask(n.runtime.Mask),
}, chain, comment); err != nil {
return nil, err
}
}
network.nets[i] = n
}
podIP, err := network.GetForwardableNetPodIP()
if err != nil {
return nil, err
}
if err := network.forwardPorts(fps, podIP); err != nil {
return nil, err
}
return &network, nil
}
// kvmSetup prepare new Networking to be used in kvm environment based on tuntap pair interfaces
// to allow communication with virtual machine created by lkvm tool
func kvmSetup(podRoot string, podID types.UUID, fps []ForwardedPort, netList common.NetList, localConfig string) (*Networking, error) {
network := Networking{
podEnv: podEnv{
podRoot: podRoot,
podID: podID,
netsLoadList: netList,
localConfig: localConfig,
},
}
var e error
// If there's a network set as default in CNI configuration
defaultGatewaySet := false
_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
if err != nil {
return nil, errwrap.Wrap(errors.New("error when parsing net address"), err)
}
network.nets, e = network.loadNets()
if e != nil {
return nil, errwrap.Wrap(errors.New("error loading network definitions"), e)
}
for i, n := range network.nets {
if n.conf.Type == "flannel" {
if err := kvmTransformFlannelNetwork(&n); err != nil {
return nil, errwrap.Wrap(errors.New("cannot transform flannel network into basic network"), err)
}
}
switch n.conf.Type {
case "ptp":
link, err := setupTapDevice(podID)
if err != nil {
return nil, err
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
// add address to host tap device
err = ensureHasAddr(
link,
&net.IPNet{
IP: n.runtime.IP4.Gateway,
Mask: net.IPMask(n.runtime.Mask),
},
)
if err != nil {
return nil, errwrap.Wrap(fmt.Errorf("cannot add address to host tap device %q", ifName), err)
}
if err := removeAllRoutesOnLink(link); err != nil {
return nil, errwrap.Wrap(fmt.Errorf("cannot remove route on host tap device %q", ifName), err)
}
if err := addRoute(link, n.runtime.IP); err != nil {
return nil, errwrap.Wrap(errors.New("cannot add on host direct route to pod"), err)
}
case "bridge":
config := BridgeNetConf{
NetConf: NetConf{
MTU: defaultMTU,
},
BrName: defaultBrName,
}
if err := json.Unmarshal(n.confBytes, &config); err != nil {
return nil, errwrap.Wrap(fmt.Errorf("error parsing %q result", n.conf.Name), err)
}
br, err := ensureBridgeIsUp(config.BrName, config.MTU)
if err != nil {
return nil, errwrap.Wrap(errors.New("error in time of bridge setup"), err)
}
link, err := setupTapDevice(podID)
if err != nil {
return nil, errwrap.Wrap(errors.New("can not setup tap device"), err)
}
err = netlink.LinkSetMaster(link, br)
if err != nil {
rErr := tuntap.RemovePersistentIface(n.runtime.IfName, tuntap.Tap)
if rErr != nil {
stderr.PrintE("warning: could not cleanup tap interface", rErr)
}
return nil, errwrap.Wrap(errors.New("can not add tap interface to bridge"), err)
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
if n.conf.IsDefaultGateway {
n.runtime.IP4.Routes = append(
n.runtime.IP4.Routes,
cnitypes.Route{Dst: *defaultNet, GW: n.runtime.IP4.Gateway},
)
defaultGatewaySet = true
config.IsGw = true
}
if config.IsGw {
err = ensureHasAddr(
br,
&net.IPNet{
IP: n.runtime.IP4.Gateway,
Mask: net.IPMask(n.runtime.Mask),
},
)
if err != nil {
return nil, errwrap.Wrap(fmt.Errorf("cannot add address to host bridge device %q", br.Name), err)
}
}
case "macvlan":
config := MacVTapNetConf{}
if err := json.Unmarshal(n.confBytes, &config); err != nil {
return nil, errwrap.Wrap(fmt.Errorf("error parsing %q result", n.conf.Name), err)
}
link, err := setupMacVTapDevice(podID, config, i)
if err != nil {
return nil, err
}
ifName := link.Attrs().Name
n.runtime.IfName = ifName
err = kvmSetupNetAddressing(&network, n, ifName)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("network %q have unsupported type: %q", n.conf.Name, n.conf.Type)
}
// Check if there's any other network set as default gateway
if defaultGatewaySet {
for _, route := range n.runtime.IP4.Routes {
if (defaultNet.String() == route.Dst.String()) && !n.conf.IsDefaultGateway {
return nil, fmt.Errorf("flannel config enables default gateway and IPAM sets default gateway via %q", n.runtime.IP4.Gateway)
}
}
}
if n.conf.IPMasq {
chain := cniutils.FormatChainName(n.conf.Name, podID.String())
comment := cniutils.FormatChainName(n.conf.Name, podID.String())
if err := ip.SetupIPMasq(&net.IPNet{
IP: n.runtime.IP,
Mask: net.IPMask(n.runtime.Mask),
}, chain, comment); err != nil {
return nil, err
}
}
network.nets[i] = n
}
podIP, err := network.GetForwardableNetPodIP()
if err != nil {
return nil, err
}
if err := network.forwardPorts(fps, podIP); err != nil {
return nil, err
}
return &network, nil
}