// ConfigureIface takes the result of IPAM plugin and
// applies to the ifName interface
func ConfigureIface(ifName string, res *Result) error {
link, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to lookup %q: %v", ifName, err)
}
if err := netlink.LinkSetUp(link); err != nil {
return fmt.Errorf("failed to set %q UP: %v", ifName, err)
}
// TODO(eyakubovich): IPv6
addr := &netlink.Addr{IPNet: &res.IP4.IP, Label: ""}
if err = netlink.AddrAdd(link, addr); err != nil {
return fmt.Errorf("failed to add IP addr to %q: %v", ifName, err)
}
for _, r := range res.IP4.Routes {
gw := r.GW
if gw == nil {
gw = res.IP4.Gateway
}
if err = ip.AddRoute(&r.Dst, gw, link); err != nil {
// we skip over duplicate routes as we assume the first one wins
if !os.IsExist(err) {
return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, ifName, err)
}
}
}
return nil
}
func setupHostVeth(vethName string, ipConf *types.IPConfig) error {
// hostVeth moved namespaces and may have a new ifindex
veth, err := netlink.LinkByName(vethName)
if err != nil {
return fmt.Errorf("failed to lookup %q: %v", vethName, err)
}
// TODO(eyakubovich): IPv6
ipn := &net.IPNet{
IP: ipConf.Gateway,
Mask: net.CIDRMask(32, 32),
}
addr := &netlink.Addr{IPNet: ipn, Label: ""}
if err = netlink.AddrAdd(veth, addr); err != nil {
return fmt.Errorf("failed to add IP addr (%#v) to veth: %v", ipn, err)
}
ipn = &net.IPNet{
IP: ipConf.IP.IP,
Mask: net.CIDRMask(32, 32),
}
// dst happens to be the same as IP/net of host veth
if err = ip.AddHostRoute(ipn, nil, veth); err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to add route on host: %v", err)
}
return nil
}
// SetupVeth sets up a virtual ethernet link.
// Should be in container netns.
func SetupVeth(contVethName string, mtu int, hostNS *os.File) (hostVeth, contVeth netlink.Link, err error) {
var hostVethName string
hostVethName, contVeth, err = makeVeth(contVethName, mtu)
if err != nil {
return
}
if err = netlink.LinkSetUp(contVeth); err != nil {
err = fmt.Errorf("failed to set %q up: %v", contVethName, err)
return
}
hostVeth, err = netlink.LinkByName(hostVethName)
if err != nil {
err = fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
return
}
if err = netlink.LinkSetUp(hostVeth); err != nil {
err = fmt.Errorf("failed to set %q up: %v", contVethName, err)
return
}
if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
err = fmt.Errorf("failed to move veth to host netns: %v", err)
return
}
return
}
// SetupVeth sets up a virtual ethernet link.
// Should be in container netns.
// TODO(eyakubovich): get rid of entropy and ask kernel to pick name via pattern
func SetupVeth(entropy, contVethName string, mtu int, hostNS *os.File) (hostVeth, contVeth netlink.Link, err error) {
// NetworkManager (recent versions) will ignore veth devices that start with "veth"
hostVethName := RandomVethName(entropy)
hostVeth, err = makeVeth(hostVethName, contVethName, mtu)
if err != nil {
err = fmt.Errorf("failed to make veth pair: %v", err)
return
}
if err = netlink.LinkSetUp(hostVeth); err != nil {
err = fmt.Errorf("failed to set %q up: %v", hostVethName, err)
return
}
contVeth, err = netlink.LinkByName(contVethName)
if err != nil {
err = fmt.Errorf("failed to lookup %q: %v", contVethName, err)
return
}
if err = netlink.LinkSetUp(contVeth); err != nil {
err = fmt.Errorf("failed to set %q up: %v", contVethName, err)
return
}
if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
err = fmt.Errorf("failed to move veth to host netns: %v", err)
return
}
return
}
func setupVeth(netns string, br *netlink.Bridge, ifName string, mtu int, pr *plugin.Result) error {
var hostVethName string
err := ns.WithNetNSPath(netns, func(hostNS *os.File) error {
// create the veth pair in the container and move host end into host netns
hostVeth, _, err := ip.SetupVeth(netns, ifName, mtu, hostNS)
if err != nil {
return err
}
if err = plugin.ConfigureIface(ifName, pr); err != nil {
return err
}
hostVethName = hostVeth.Attrs().Name
return nil
})
if err != nil {
return err
}
// need to lookup hostVeth again as its index has changed during ns move
hostVeth, err := netlink.LinkByName(hostVethName)
if err != nil {
return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
}
// connect host veth end to the bridge
if err = netlink.LinkSetMaster(hostVeth, br); err != nil {
return fmt.Errorf("failed to connect %q to bridge %v: %v", hostVethName, br.Attrs().Name, err)
}
return nil
}
func createIpvlan(conf *NetConf, ifName string, netns *os.File) error {
mode, err := modeFromString(conf.Mode)
if err != nil {
return err
}
m, err := netlink.LinkByName(conf.Master)
if err != nil {
return fmt.Errorf("failed to lookup master %q: %v", conf.Master, err)
}
mv := &netlink.IPVlan{
LinkAttrs: netlink.LinkAttrs{
MTU: conf.MTU,
Name: ifName,
ParentIndex: m.Attrs().Index,
Namespace: netlink.NsFd(int(netns.Fd())),
},
Mode: mode,
}
if err := netlink.LinkAdd(mv); err != nil {
return fmt.Errorf("failed to create ipvlan: %v", err)
}
return err
}
func renameLink(curName, newName string) error {
link, err := netlink.LinkByName(curName)
if err != nil {
return err
}
return netlink.LinkSetName(link, newName)
}
func bridgeByName(name string) (*netlink.Bridge, error) {
l, err := netlink.LinkByName(name)
if err != nil {
return nil, fmt.Errorf("could not lookup %q: %v", name, err)
}
br, ok := l.(*netlink.Bridge)
if !ok {
return nil, fmt.Errorf("%q already exists but is not a bridge", name)
}
return br, nil
}
func loUp() error {
lo, err := netlink.LinkByName("lo")
if err != nil {
return fmt.Errorf("failed to lookup lo: %v", err)
}
if err := netlink.LinkSetUp(lo); err != nil {
return fmt.Errorf("failed to set lo up: %v", err)
}
return nil
}
// DelLinkByName removes an interface link.
func DelLinkByName(ifName string) error {
iface, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to lookup %q: %v", ifName, err)
}
if err = netlink.LinkDel(iface); err != nil {
return fmt.Errorf("failed to delete %q: %v", ifName, err)
}
return nil
}
func getDefaultGW(family int) (string, error) {
l, err := netlink.LinkByName("lo")
if err != nil {
return "", err
}
routes, err := netlink.RouteList(l, family)
if err != nil {
return "", err
}
return routes[0].Gw.String(), nil
}
func (an activeNet) IfName() string {
if an.conf.Type == "macvlan" {
// macvtap device passed as parameter to lkvm binary have different
// kind of name, path to /dev/tapN made with N as link index
link, err := netlink.LinkByName(an.runtime.IfName)
if err != nil {
log.Printf("Cannot get interface '%v': %v", an.runtime.IfName, err)
return ""
}
return fmt.Sprintf("/dev/tap%d", link.Attrs().Index)
}
return an.runtime.IfName
}
// setupTapDevice creates persistent tap devices
// and returns a newly created netlink.Link structure
func setupTapDevice() (netlink.Link, error) {
ifName, err := tuntap.CreatePersistentIface(tuntap.Tap)
if err != nil {
return nil, fmt.Errorf("tuntap persist %v", err)
}
link, err := netlink.LinkByName(ifName)
if err != nil {
return nil, fmt.Errorf("cannot find link %q: %v", ifName, err)
}
err = netlink.LinkSetUp(link)
if err != nil {
return nil, fmt.Errorf("cannot set link up %q: %v", ifName, err)
}
return link, nil
}
// DelLinkByNameAddr remove an interface returns its IP address
// of the specified family
func DelLinkByNameAddr(ifName string, family int) (*net.IPNet, error) {
iface, err := netlink.LinkByName(ifName)
if err != nil {
return nil, fmt.Errorf("failed to lookup %q: %v", ifName, err)
}
addrs, err := netlink.AddrList(iface, family)
if err != nil || len(addrs) == 0 {
return nil, fmt.Errorf("failed to get IP addresses for %q: %v", ifName, err)
}
if err = netlink.LinkDel(iface); err != nil {
return nil, fmt.Errorf("failed to delete %q: %v", ifName, err)
}
return addrs[0].IPNet, nil
}
// setupTapDevice creates persistent tap devices
// and returns a newly created netlink.Link structure
func setupTapDevice(podID types.UUID) (netlink.Link, error) {
// network device names are limited to 16 characters
// the suffix %d will be replaced by the kernel with a suitable number
nameTemplate := fmt.Sprintf("rkt-%s-tap%%d", podID.String()[0:4])
ifName, err := tuntap.CreatePersistentIface(nameTemplate, tuntap.Tap)
if err != nil {
return nil, fmt.Errorf("tuntap persist %v", err)
}
link, err := netlink.LinkByName(ifName)
if err != nil {
return nil, fmt.Errorf("cannot find link %q: %v", ifName, err)
}
err = netlink.LinkSetUp(link)
if err != nil {
return nil, fmt.Errorf("cannot set link up %q: %v", ifName, err)
}
return link, nil
}
// teardownKvmNets teardown every active networking from networking by
// removing tuntap interface and releasing its ip from IPAM plugin
func (n *Networking) teardownKvmNets() {
for _, an := range n.nets {
switch an.conf.Type {
case "ptp", "bridge":
// remove tuntap interface
tuntap.RemovePersistentIface(an.runtime.IfName, tuntap.Tap)
case "macvlan":
link, err := netlink.LinkByName(an.runtime.IfName)
if err != nil {
log.Printf("Cannot find link `%v`: %v", an.runtime.IfName, err)
continue
} else {
err := netlink.LinkDel(link)
if err != nil {
log.Printf("Cannot remove link `%v`: %v", an.runtime.IfName, err)
continue
}
}
default:
log.Printf("Unsupported network type: %q", an.conf.Type)
continue
}
// ugly hack again to directly call IPAM plugin to release IP
an.conf.Type = an.conf.IPAM.Type
_, err := n.execNetPlugin("DEL", &an, an.runtime.IfName)
if err != nil {
log.Printf("Error executing network plugin: %q", err)
}
// remove masquerading if it was prepared
if an.conf.IPMasq {
chain := getChainName(n.podID.String(), an.conf.Name)
err := ip.TeardownIPMasq(&net.IPNet{
IP: an.runtime.IP,
Mask: net.IPMask(an.runtime.Mask),
}, chain)
if err != nil {
log.Printf("Error on removing masquerading: %q", err)
}
}
}
}
// setupTapDevice creates persistent macvtap device
// and returns a newly created netlink.Link structure
func setupMacVTapDevice(podID types.UUID, config MacVTapNetConf) (netlink.Link, error) {
master, err := netlink.LinkByName(config.Master)
if err != nil {
return nil, fmt.Errorf("Cannot find master device '%v': %v", config.Master, err)
}
var mode netlink.MacvlanMode
switch config.Mode {
// if not set - defaults to bridge mode as in:
// https://github.com/coreos/rkt/blob/master/Documentation/networking.md#macvlan
case "", "bridge":
mode = netlink.MACVLAN_MODE_BRIDGE
case "private":
mode = netlink.MACVLAN_MODE_PRIVATE
case "vepa":
mode = netlink.MACVLAN_MODE_VEPA
case "passthru":
mode = netlink.MACVLAN_MODE_PASSTHRU
default:
return nil, fmt.Errorf("Unsupported macvtap mode: %v", config.Mode)
}
mtu := master.Attrs().MTU
if config.MTU != 0 {
mtu = config.MTU
}
nameTemplate := fmt.Sprintf("rkt-%s-vtap%%d", podID.String()[0:4])
link := &netlink.Macvtap{
Macvlan: netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: nameTemplate,
MTU: mtu,
ParentIndex: master.Attrs().Index,
},
Mode: mode,
},
}
if err := netlink.LinkAdd(link); err != nil {
return nil, fmt.Errorf("Cannot create macvtap interface: %v", err)
}
return link, nil
}
func createMacvlan(conf *NetConf, ifName string, netns *os.File) error {
mode, err := modeFromString(conf.Mode)
if err != nil {
return err
}
m, err := netlink.LinkByName(conf.Master)
if err != nil {
return fmt.Errorf("failed to lookup master %q: %v", conf.Master, err)
}
// due to kernel bug we have to create with tmpname or it might
// collide with the name on the host and error out
tmpName, err := ip.RandomVethName()
if err != nil {
return err
}
mv := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
MTU: conf.MTU,
Name: tmpName,
ParentIndex: m.Attrs().Index,
Namespace: netlink.NsFd(int(netns.Fd())),
},
Mode: mode,
}
if err := netlink.LinkAdd(mv); err != nil {
return fmt.Errorf("failed to create macvlan: %v", err)
}
return ns.WithNetNS(netns, false, func(_ *os.File) error {
err := renameLink(tmpName, ifName)
if err != nil {
return fmt.Errorf("failed to rename macvlan to %q: %v", ifName, err)
}
return nil
})
}
// AcquireLease gets an DHCP lease and then maintains it in the background
// by periodically renewing it. The acquired lease can be released by
// calling DHCPLease.Stop()
func AcquireLease(clientID, netns, ifName string) (*DHCPLease, error) {
errCh := make(chan error, 1)
l := &DHCPLease{
clientID: clientID,
stop: make(chan struct{}),
}
log.Printf("%v: acquiring lease", clientID)
l.wg.Add(1)
go func() {
errCh <- ns.WithNetNSPath(netns, true, func(_ *os.File) error {
defer l.wg.Done()
link, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("error looking up %q: %v", ifName, err)
}
l.link = link
if err = l.acquire(); err != nil {
return err
}
log.Printf("%v: lease acquired, expiration is %v", l.clientID, l.expireTime)
errCh <- nil
l.maintain()
return nil
})
}()
if err := <-errCh; err != nil {
return nil, err
}
return l, nil
}
func setupContainerVeth(netns, ifName string, mtu int, pr *types.Result) (string, error) {
// The IPAM result will be something like IP=192.168.3.5/24, GW=192.168.3.1.
// What we want is really a point-to-point link but veth does not support IFF_POINTOPONT.
// Next best thing would be to let it ARP but set interface to 192.168.3.5/32 and
// add a route like "192.168.3.0/24 via 192.168.3.1 dev $ifName".
// Unfortunately that won't work as the GW will be outside the interface's subnet.
// Our solution is to configure the interface with 192.168.3.5/24, then delete the
// "192.168.3.0/24 dev $ifName" route that was automatically added. Then we add
// "192.168.3.1/32 dev $ifName" and "192.168.3.0/24 via 192.168.3.1 dev $ifName".
// In other words we force all traffic to ARP via the gateway except for GW itself.
var hostVethName string
err := ns.WithNetNSPath(netns, false, func(hostNS *os.File) error {
hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
if err != nil {
return err
}
if err = ipam.ConfigureIface(ifName, pr); err != nil {
return err
}
contVeth, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to look up %q: %v", ifName, err)
}
// Delete the route that was automatically added
route := netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.IP.IP.Mask(pr.IP4.IP.Mask),
Mask: pr.IP4.IP.Mask,
},
Scope: netlink.SCOPE_NOWHERE,
}
if err := netlink.RouteDel(&route); err != nil {
return fmt.Errorf("failed to delete route %v: %v", route, err)
}
for _, r := range []netlink.Route{
netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.Gateway,
Mask: net.CIDRMask(32, 32),
},
Scope: netlink.SCOPE_LINK,
Src: pr.IP4.IP.IP,
},
netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Dst: &net.IPNet{
IP: pr.IP4.IP.IP.Mask(pr.IP4.IP.Mask),
Mask: pr.IP4.IP.Mask,
},
Scope: netlink.SCOPE_UNIVERSE,
Gw: pr.IP4.Gateway,
Src: pr.IP4.IP.IP,
},
} {
if err := netlink.RouteAdd(&r); err != nil {
return fmt.Errorf("failed to add route %v: %v", r, err)
}
}
hostVethName = hostVeth.Attrs().Name
return nil
})
return hostVethName, err
}