func ensureEgressFd(link netlink.Link, fd int) error {
q, err := ensureQdisc(link, "fq_codel", netlink.MakeHandle(1, 0), netlink.HANDLE_ROOT)
if err != nil {
return err
}
fHandle := netlink.MakeHandle(0, 2)
filter := &netlink.U32{
FilterAttrs: netlink.FilterAttrs{
LinkIndex: link.Attrs().Index,
Parent: q.Attrs().Handle,
Priority: 1,
Protocol: syscall.ETH_P_ALL,
},
Actions: []netlink.Action{
&netlink.BpfAction{Fd: fd, Name: "bpf1"},
},
ClassId: fHandle,
}
filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_EGRESS)
if err != nil {
return fmt.Errorf("failed fetching egress filter list: %s", err)
}
for _, f := range filters {
if f, ok := f.(*netlink.U32); ok {
if f.ClassId == fHandle {
return nil
}
}
}
if err := netlink.FilterAdd(filter); err != nil {
return fmt.Errorf("failed adding egress filter: %v", err)
}
return nil
}
func getDynamicIP(t Netlink, link netlink.Link, dc client.Client) (*dhcp.Packet, error) {
var ack *dhcp.Packet
var err error
// use dhcp to acquire address
id, err := client.NewID(link.Attrs().Index, link.Attrs().HardwareAddr)
if err != nil {
return nil, err
}
ack, err = dc.Request(id)
if err != nil {
log.Errorf("error sending dhcp request: %s", err)
return nil, err
}
if ack.YourIP() == nil || ack.SubnetMask() == nil {
err = fmt.Errorf("dhcp assigned nil ip or subnet mask")
log.Error(err)
return nil, err
}
log.Infof("DHCP response: IP=%s, SubnetMask=%s, Gateway=%s, DNS=%s, Lease Time=%s", ack.YourIP(), ack.SubnetMask(), ack.Gateway(), ack.DNS(), ack.LeaseTime())
defer func() {
if err != nil && ack != nil {
dc.Release(ack)
}
}()
return ack, nil
}
// delRouteIface clean up the required L3 mode default ns route
func delRouteIface(ipVlanL3Network *net.IPNet, iface netlink.Link) error {
return netlink.RouteDel(&netlink.Route{
LinkIndex: iface.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: ipVlanL3Network,
})
}
func setFqCodelFd(iface netlink.Link, path string) error {
fd, err := netlink.BpfOpen(path)
if err != nil {
return fmt.Errorf("failed loading bpf program %v", err)
}
defer syscall.Close(fd)
fq := &netlink.GenericQdisc{
QdiscAttrs: netlink.QdiscAttrs{
LinkIndex: iface.Attrs().Index,
Handle: netlink.MakeHandle(1, 0),
Parent: netlink.HANDLE_ROOT,
},
QdiscType: "fq_codel",
}
if err := netlink.QdiscAdd(fq); err != nil {
return fmt.Errorf("failed setting egress qdisc: %v", err)
}
u32 := &netlink.U32{
FilterAttrs: netlink.FilterAttrs{
LinkIndex: iface.Attrs().Index,
Parent: fq.QdiscAttrs.Handle,
Protocol: syscall.ETH_P_ALL,
//Handle: 10,
//Priority: 10,
},
ClassId: netlink.MakeHandle(1, 2),
BpfFd: fd,
}
if err := netlink.FilterAdd(u32); err != nil {
return fmt.Errorf("failed adding egress filter: %v", err)
}
return nil
}
func (u *NetLinkProbe) onLinkDeleted(link netlink.Link) {
index := link.Attrs().Index
u.Graph.Lock()
defer u.Graph.Unlock()
intf := u.Graph.LookupFirstChild(u.Root, graph.Metadata{"IfIndex": index})
// case of removing the interface from a bridge
if intf != nil {
parents := u.Graph.LookupParents(intf, graph.Metadata{"Type": "bridge"})
for _, parent := range parents {
u.Graph.Unlink(parent, intf)
}
}
// check whether the interface has been deleted or not
// we get a delete event when an interace is removed from a bridge
_, err := u.netlink.LinkByIndex(index)
if err != nil && intf != nil {
// if openvswitch do not remove let's do the job by ovs piece of code
if intf.Metadata()["Driver"] == "openvswitch" && intf.Metadata()["UUID"] != "" {
u.Graph.Unlink(u.Root, intf)
} else {
u.Graph.DelNode(intf)
}
}
delete(u.indexToChildrenQueue, int64(index))
}
func linkAddrUpdate(old, new *net.IPNet, t Netlink, link netlink.Link) error {
log.Infof("setting ip address %s for link %s", new, link.Attrs().Name)
if old != nil && !old.IP.Equal(new.IP) {
log.Debugf("removing old address %s", old)
if err := t.AddrDel(link, &netlink.Addr{IPNet: old}); err != nil {
if errno, ok := err.(syscall.Errno); !ok || errno != syscall.EADDRNOTAVAIL {
log.Errorf("failed to remove existing address %s: %s", old, err)
return err
}
}
log.Debugf("removed old address %s for link %s", old, link.Attrs().Name)
}
// assign IP to NIC
if err := t.AddrAdd(link, &netlink.Addr{IPNet: new}); err != nil {
if errno, ok := err.(syscall.Errno); !ok || errno != syscall.EEXIST {
log.Errorf("failed to assign ip %s for link %s", new, link.Attrs().Name)
return err
}
log.Warnf("address %s already set on interface %s", new, link.Attrs().Name)
}
log.Debugf("added address %s to link %s", new, link.Attrs().Name)
return nil
}
// addRouteIface required for L3 mode adds a link scoped route in the default ns
func AddRouteIface(ipVlanL3Network *net.IPNet, iface netlink.Link) error {
return netlink.RouteAdd(&netlink.Route{
LinkIndex: iface.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: ipVlanL3Network,
})
}
func updateDefaultRoute(t Netlink, link netlink.Link, endpoint *NetworkEndpoint) error {
// Add routes
if !endpoint.Network.Default || ip.IsUnspecifiedIP(endpoint.Network.Gateway.IP) {
log.Debugf("not setting route for network: default=%v gateway=%s", endpoint.Network.Default, endpoint.Network.Gateway.IP)
return nil
}
_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")
// delete default route first
if err := t.RouteDel(&netlink.Route{LinkIndex: link.Attrs().Index, Dst: defaultNet}); err != nil {
if errno, ok := err.(syscall.Errno); !ok || errno != syscall.ESRCH {
return fmt.Errorf("could not update default route: %s", err)
}
}
log.Infof("Setting default gateway to %s", endpoint.Network.Gateway.IP)
route := &netlink.Route{LinkIndex: link.Attrs().Index, Dst: defaultNet, Gw: endpoint.Network.Gateway.IP}
if err := t.RouteAdd(route); err != nil {
detail := fmt.Sprintf("failed to add gateway route for endpoint %s: %s", endpoint.Network.Name, err)
return errors.New(detail)
}
log.Infof("updated default route to %s interface, gateway: %s", endpoint.Network.Name, endpoint.Network.Gateway.IP)
return nil
}
func (u *NetLinkProbe) addGenericLinkToTopology(link netlink.Link, m graph.Metadata) *graph.Node {
name := link.Attrs().Name
index := int64(link.Attrs().Index)
var intf *graph.Node
if name != "lo" {
intf = u.Graph.LookupFirstChild(u.Root, graph.Metadata{
"IfIndex": index,
})
}
if intf == nil {
intf = u.Graph.NewNode(graph.GenID(), m)
}
if intf == nil {
return nil
}
if !u.Graph.AreLinked(u.Root, intf) {
u.Graph.Link(u.Root, intf)
}
u.handleIntfIsChild(intf, link)
u.handleIntfIsVeth(intf, link)
u.handleIntfIsBond(intf, link)
return intf
}
func addIPConfigToLink(ipConfig *ipam.IPConfig, link netlink.Link, ifName string) error {
log.Debugf("Configuring link %+v/%s with %+v", link, ifName, ipConfig)
addr := &netlink.Addr{IPNet: &ipConfig.IP}
if err := netlink.AddrAdd(link, addr); err != nil {
return fmt.Errorf("failed to add addr to %q: %v", ifName, err)
}
// Sort provided routes to make sure we apply any more specific
// routes first which may be used as nexthops in wider routes
sort.Sort(ipam.ByMask(ipConfig.Routes))
for _, r := range ipConfig.Routes {
log.Debugf("Adding route %+v", r)
rt := &netlink.Route{
LinkIndex: link.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: &r.Destination,
Gw: r.NextHop,
}
if r.IsL2() {
rt.Scope = netlink.SCOPE_LINK
}
if err := netlink.RouteAdd(rt); err != nil {
if !os.IsExist(err) {
return fmt.Errorf("failed to add route '%s via %v dev %v': %v",
r.Destination.String(), r.NextHop, ifName, err)
}
}
}
return nil
}
func setIngressFd(iface netlink.Link, path string) error {
fd, err := netlink.BpfOpen(path)
if err != nil {
return fmt.Errorf("failed loading bpf program %v", err)
}
defer syscall.Close(fd)
ingress := &netlink.Ingress{
QdiscAttrs: netlink.QdiscAttrs{
LinkIndex: iface.Attrs().Index,
Handle: netlink.MakeHandle(0xffff, 0),
Parent: netlink.HANDLE_INGRESS,
},
}
if err := netlink.QdiscAdd(ingress); err != nil {
return fmt.Errorf("failed setting ingress qdisc: %v", err)
}
u32 := &netlink.U32{
FilterAttrs: netlink.FilterAttrs{
LinkIndex: iface.Attrs().Index,
Parent: ingress.QdiscAttrs.Handle,
Priority: 1,
Protocol: syscall.ETH_P_ALL,
},
ClassId: netlink.MakeHandle(1, 1),
BpfFd: fd,
}
if err := netlink.FilterAdd(u32); err != nil {
return fmt.Errorf("failed adding ingress filter: %v", err)
}
return nil
}
func (t *Mocker) LinkSetUp(link netlink.Link) error {
defer trace.End(trace.Begin(fmt.Sprintf("Bringing %s up", link.Attrs().Name)))
iface := link.(*Interface)
iface.Up = true
return nil
}
func (t *Mocker) LinkSetAlias(link netlink.Link, alias string) error {
defer trace.End(trace.Begin(fmt.Sprintf("Adding alias %s to %s", alias, link.Attrs().Name)))
iface := link.(*Interface)
iface.Alias = alias
return nil
}
func (t *Mocker) LinkSetDown(link netlink.Link) error {
defer trace.End(trace.Begin(fmt.Sprintf("Bringing %s down", link.Attrs().Name)))
iface := link.(*Interface)
iface.Up = false
// TODO: should this drop addresses?
return nil
}
// delRemoteRoute deletes a host-scoped route to a device.
func delRemoteRoute(neighborNetwork *net.IPNet, nextHop net.IP, iface netlink.Link) error {
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: iface.Attrs().Index,
Dst: neighborNetwork,
Gw: nextHop,
})
}
// delRoute deletes any netlink route
func delRoute(route netlink.Route, iface netlink.Link) error {
return netlink.RouteDel(&netlink.Route{
Scope: route.Scope,
LinkIndex: iface.Attrs().Index,
Dst: route.Dst,
Gw: route.Gw,
})
}
// AddHostRoute adds a host-scoped route to a device.
func AddHostRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error {
return netlink.RouteAdd(&netlink.Route{
LinkIndex: dev.Attrs().Index,
Scope: netlink.SCOPE_HOST,
Dst: ipn,
Gw: gw,
})
}
func (rm *RouteManager) AddRoute(link netlink.Link, network *net.IPNet, gateway net.IP) error {
return rm.Netlinker.RouteAdd(&netlink.Route{
LinkIndex: link.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: network,
Gw: gateway,
})
}
// delRemoteRoute deletes a host-scoped route to a device.
func delRemoteRoute(neighborNetwork *net.IPNet, nextHop net.IP, iface netlink.Link) error {
log.Debugf("Deleting route: IP Prefix: [ %s ] - Next Hop: [ %s ] - Source Interface: [ %s ]", neighborNetwork, nextHop, iface.Attrs().Name)
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: iface.Attrs().Index,
Dst: neighborNetwork,
Gw: nextHop,
})
}
func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, options ...NeighOption) error {
nh := n.findNeighbor(dstIP, dstMac)
if nh != nil {
// If it exists silently return
return nil
}
nh = &neigh{
dstIP: dstIP,
dstMac: dstMac,
}
nh.processNeighOptions(options...)
if nh.linkName != "" {
nh.linkDst = n.findDst(nh.linkName, false)
if nh.linkDst == "" {
return fmt.Errorf("could not find the interface with name %s", nh.linkName)
}
}
return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error {
var iface netlink.Link
if nh.linkDst != "" {
var err error
iface, err = netlink.LinkByName(nh.linkDst)
if err != nil {
return fmt.Errorf("could not find interface with destination name %s: %v",
nh.linkDst, err)
}
}
nlnh := &netlink.Neigh{
IP: dstIP,
HardwareAddr: dstMac,
State: netlink.NUD_PERMANENT,
Family: nh.family,
}
if nlnh.Family > 0 {
nlnh.Flags = netlink.NTF_SELF
}
if nh.linkDst != "" {
nlnh.LinkIndex = iface.Attrs().Index
}
if err := netlink.NeighSet(nlnh); err != nil {
return fmt.Errorf("could not add neighbor entry: %v", err)
}
n.neighbors = append(n.neighbors, nh)
return nil
})
}
func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr, osDelete bool) error {
var (
iface netlink.Link
err error
)
nh := n.findNeighbor(dstIP, dstMac)
if nh == nil {
return fmt.Errorf("could not find the neighbor entry to delete")
}
if osDelete {
n.Lock()
nlh := n.nlHandle
n.Unlock()
if nh.linkDst != "" {
iface, err = nlh.LinkByName(nh.linkDst)
if err != nil {
return fmt.Errorf("could not find interface with destination name %s: %v",
nh.linkDst, err)
}
}
nlnh := &netlink.Neigh{
IP: dstIP,
State: netlink.NUD_PERMANENT,
Family: nh.family,
}
if nlnh.Family > 0 {
nlnh.HardwareAddr = dstMac
nlnh.Flags = netlink.NTF_SELF
}
if nh.linkDst != "" {
nlnh.LinkIndex = iface.Attrs().Index
}
if err := nlh.NeighDel(nlnh); err != nil {
return fmt.Errorf("could not delete neighbor entry: %v", err)
}
}
n.Lock()
for i, nh := range n.neighbors {
if nh.dstIP.Equal(dstIP) && bytes.Equal(nh.dstMac, dstMac) {
n.neighbors = append(n.neighbors[:i], n.neighbors[i+1:]...)
break
}
}
n.Unlock()
return nil
}
// delRoute deletes any netlink route
func delRoute(route netlink.Route, iface netlink.Link) error {
log.Debugf("Deleting route in the default namespace for IPVlan L3 mode with the following:")
log.Debugf("IP Prefix: [ %s ] - Next Hop: [ %s ] - Source Interface: [ %s ]", route.Dst, route.Gw, iface.Attrs().Name)
return netlink.RouteDel(&netlink.Route{
Scope: route.Scope,
LinkIndex: iface.Attrs().Index,
Dst: route.Dst,
Gw: route.Gw,
})
}
func routeAdd(ip *net.IPNet, iface netlink.Link) error {
route := netlink.Route{
LinkIndex: iface.Attrs().Index,
Dst: ip,
}
log.Debugf("Adding route %+v", route)
if err := netlink.RouteAdd(&route); err != nil {
log.Errorf("Unable to add route %+v: %+v", route, err)
}
return nil
}
func addRoute(link netlink.Link, podIP net.IP) error {
route := netlink.Route{
LinkIndex: link.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: &net.IPNet{
IP: podIP,
Mask: net.IPv4Mask(0xff, 0xff, 0xff, 0xff),
},
}
return netlink.RouteAdd(&route)
}
func NewVlan(link netlink.Link, name string, id int) (*Vlan, error) {
if name == "" {
name = fmt.Sprintf("%s.%d", link.Attrs().Name, id)
}
v := &Vlan{
name: name,
link: link,
id: id,
}
return v, v.init()
}
func applyNetConf(link netlink.Link, netConf InterfaceConfig) error {
if netConf.IPV4LL {
if err := AssignLinkLocalIP(link); err != nil {
log.Errorf("IPV4LL set failed: %v", err)
return err
}
} else if netConf.Address == "" {
return nil
} else {
addr, err := netlink.ParseAddr(netConf.Address)
if err != nil {
return err
}
if err := netlink.AddrAdd(link, addr); err != nil {
//Ignore this error
log.Errorf("addr add failed: %v", err)
} else {
log.Infof("Set %s on %s", netConf.Address, link.Attrs().Name)
}
}
if netConf.MTU > 0 {
if err := netlink.LinkSetMTU(link, netConf.MTU); err != nil {
log.Errorf("set MTU Failed: %v", err)
return err
}
}
if err := netlink.LinkSetUp(link); err != nil {
log.Errorf("failed to setup link: %v", err)
return err
}
if netConf.Gateway != "" {
gatewayIp := net.ParseIP(netConf.Gateway)
if gatewayIp == nil {
return errors.New("Invalid gateway address " + netConf.Gateway)
}
route := netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
Gw: net.ParseIP(netConf.Gateway),
}
if err := netlink.RouteAdd(&route); err != nil {
log.Errorf("gateway set failed: %v", err)
return err
}
log.Infof("Set default gateway %s", netConf.Gateway)
}
return nil
}
// AddHostRoute adds a host-scoped route to a device.
func AddRoute(neighborNetwork *net.IPNet, nextHop net.IP, iface netlink.Link) error {
log.Debugf("Adding route in the default namespace for IPVlan L3 mode with the following:")
log.Debugf("IP Prefix: [ %s ] - Next Hop: [ %s ] - Source Interface: [ %s ]",
neighborNetwork, nextHop, iface.Attrs().Name)
return netlink.RouteAdd(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: iface.Attrs().Index,
Dst: neighborNetwork,
Gw: nextHop,
})
}
func AddRoute(link netlink.Link, scope netlink.Scope, dst *net.IPNet, gw net.IP) error {
err := netlink.RouteAdd(&netlink.Route{
LinkIndex: link.Attrs().Index,
Scope: scope,
Dst: dst,
Gw: gw,
})
if os.IsExist(err) { // squash duplicate route errors
err = nil
}
return err
}
func linkToNetDev(link netlink.Link) (NetDev, error) {
addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
if err != nil {
return NetDev{}, err
}
netDev := NetDev{Name: link.Attrs().Name, MAC: link.Attrs().HardwareAddr}
for _, addr := range addrs {
netDev.CIDRs = append(netDev.CIDRs, addr.IPNet)
}
return netDev, nil
}
// Adds a macvlan interface to a container for use with the egress router feature
func addMacvlan(netns string) error {
var defIface netlink.Link
var err error
// Find interface with the default route
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
if err != nil {
return fmt.Errorf("failed to read routes: %v", err)
}
for _, r := range routes {
if r.Dst == nil {
defIface, err = netlink.LinkByIndex(r.LinkIndex)
if err != nil {
return fmt.Errorf("failed to get default route interface: %v", err)
}
}
}
if defIface == nil {
return fmt.Errorf("failed to find default route interface")
}
podNs, err := ns.GetNS(netns)
if err != nil {
return fmt.Errorf("could not open netns %q", netns)
}
defer podNs.Close()
err = netlink.LinkAdd(&netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
MTU: defIface.Attrs().MTU,
Name: "macvlan0",
ParentIndex: defIface.Attrs().Index,
Namespace: netlink.NsFd(podNs.Fd()),
},
Mode: netlink.MACVLAN_MODE_PRIVATE,
})
if err != nil {
return fmt.Errorf("failed to create macvlan interface: %v", err)
}
return podNs.Do(func(netns ns.NetNS) error {
l, err := netlink.LinkByName("macvlan0")
if err != nil {
return fmt.Errorf("failed to find macvlan interface: %v", err)
}
err = netlink.LinkSetUp(l)
if err != nil {
return fmt.Errorf("failed to set macvlan interface up: %v", err)
}
return nil
})
}