func resourceLXCBridgeDelete(d *schema.ResourceData, meta interface{}) error {
bridgeIndex, err := strconv.Atoi(d.Id())
if err != nil {
return fmt.Errorf("Internal error reading resource ID: %v", err)
}
bridge, err := netlink.LinkByIndex(bridgeIndex)
if err != nil {
return fmt.Errorf("Unable to find bridge %v: %v", bridgeIndex, err)
}
links, err := netlink.LinkList()
if err != nil {
return fmt.Errorf("Error listing interfaces: %v", err)
}
bridgeEmpty := true
for _, link := range links {
if link.Attrs().MasterIndex == bridge.Attrs().Index {
bridgeEmpty = false
log.Printf("[INFO] Link %s is still attached to bridge %s", link.Attrs().Name, bridge.Attrs().Name)
}
}
if bridgeEmpty == false {
return fmt.Errorf("Unable to delete bridge %s. Interfaces are still attached to it.", bridge.Attrs().Name)
} else {
if err := netlink.LinkDel(bridge); err != nil {
return fmt.Errorf("Error deleting bridge: %s", err)
}
}
return nil
}
func testAccCheckLXCBridgeExists(t *testing.T, n string, bridge *netlink.Link) resource.TestCheckFunc {
return func(s *terraform.State) error {
rs, ok := s.RootModule().Resources[n]
if !ok {
return fmt.Errorf("Not found: %v", n)
}
if rs.Primary.ID == "" {
return fmt.Errorf("No ID is set")
}
bridgeIndex, err := strconv.Atoi(rs.Primary.ID)
if err != nil {
return fmt.Errorf("Internal error reading resource ID.")
}
br, err := netlink.LinkByIndex(bridgeIndex)
if err != nil {
return fmt.Errorf("Error searching for bridge.")
} else {
*bridge = br
return nil
}
return fmt.Errorf("Unable to find bridge.")
}
}
// gatherLinks listens for netlink notifications and collects the list of links
// created during the subscription window.
func gatherLinks(ch <-chan netlink.LinkUpdate) ([]netlink.Link, error) {
linkSet := make(map[int32]bool)
timeout := time.After(1000 * time.Millisecond)
Debug.Printf("Waiting for link updates\n")
outer:
for {
select {
case update := <-ch:
if update.Header.Type == syscall.RTM_NEWLINK {
if _, ok := update.Link.(*netlink.Veth); ok {
linkSet[update.Index] = true
}
} else if update.Header.Type == syscall.RTM_DELLINK {
delete(linkSet, update.Index)
}
case <-timeout:
break outer
}
}
links := []netlink.Link{}
for index, _ := range linkSet {
l, err := netlink.LinkByIndex(int(index))
if err == nil {
links = append(links, l)
}
}
return links, nil
}
func (u *NetLinkProbe) onLinkAdded(index int) {
link, err := netlink.LinkByIndex(index)
if err != nil {
logging.GetLogger().Warningf("Failed to find interface %d: %s", index, err.Error())
return
}
u.addLinkToTopology(link)
}
func GetBridgeFromIndex(idx int) (string, error) {
var attr, bridge *netlink.LinkAttrs
links, err := netlink.LinkList()
if err != nil {
glog.Error(err)
return "", err
}
for _, link := range links {
if link.Type() != "veth" {
continue
}
if link.Attrs().Index == idx {
attr = link.Attrs()
break
}
}
if attr == nil {
return "", fmt.Errorf("cann't find nic whose ifindex is %d", idx)
}
for _, link := range links {
if link.Type() != "bridge" && link.Type() != "openvswitch" {
continue
}
if link.Attrs().Index == attr.MasterIndex {
bridge = link.Attrs()
break
}
}
if bridge == nil {
return "", fmt.Errorf("cann't find bridge contains nic whose ifindex is %d", idx)
}
if bridge.Name == "ovs-system" {
veth, err := netlink.LinkByIndex(idx)
if err != nil {
return "", err
}
out, err := exec.Command("ovs-vsctl", "port-to-br", veth.Attrs().Name).CombinedOutput()
if err != nil {
return "", err
}
bridge.Name = strings.TrimSpace(string(out))
}
glog.Infof("find bridge %s", bridge.Name)
return bridge.Name, nil
}
// Adds a macvlan interface to a container for use with the egress router feature
func addMacvlan(netns string) error {
var defIface netlink.Link
var err error
// Find interface with the default route
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
if err != nil {
return fmt.Errorf("failed to read routes: %v", err)
}
for _, r := range routes {
if r.Dst == nil {
defIface, err = netlink.LinkByIndex(r.LinkIndex)
if err != nil {
return fmt.Errorf("failed to get default route interface: %v", err)
}
}
}
if defIface == nil {
return fmt.Errorf("failed to find default route interface")
}
podNs, err := ns.GetNS(netns)
if err != nil {
return fmt.Errorf("could not open netns %q", netns)
}
defer podNs.Close()
err = netlink.LinkAdd(&netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
MTU: defIface.Attrs().MTU,
Name: "macvlan0",
ParentIndex: defIface.Attrs().Index,
Namespace: netlink.NsFd(podNs.Fd()),
},
Mode: netlink.MACVLAN_MODE_PRIVATE,
})
if err != nil {
return fmt.Errorf("failed to create macvlan interface: %v", err)
}
return podNs.Do(func(netns ns.NetNS) error {
l, err := netlink.LinkByName("macvlan0")
if err != nil {
return fmt.Errorf("failed to find macvlan interface: %v", err)
}
err = netlink.LinkSetUp(l)
if err != nil {
return fmt.Errorf("failed to set macvlan interface up: %v", err)
}
return nil
})
}
func (u *NetLinkProbe) onLinkDeleted(index int) {
logging.GetLogger().Debugf("Link %d deleted", index)
u.Graph.Lock()
defer u.Graph.Unlock()
var intf *graph.Node
intfs := u.Graph.LookupNodes(graph.Metadata{"IfIndex": int64(index)})
switch l := len(intfs); {
case l == 1:
intf = intfs[0]
case l > 1:
Loop:
for _, i := range intfs {
parents := u.Graph.LookupParentNodes(i, nil)
for _, parent := range parents {
if parent.ID == u.Root.ID {
intf = i
break Loop
}
}
}
}
// case of removing the interface from a bridge
if intf != nil {
parents := u.Graph.LookupParentNodes(intf, graph.Metadata{"Type": "bridge"})
for _, parent := range parents {
u.Graph.Unlink(parent, intf)
}
}
// check wheter the interface has been deleted or not
// we get a delete event when an interace is removed from a bridge
_, err := netlink.LinkByIndex(index)
if err != nil && intf != nil {
if driver, ok := intf.Metadata()["Driver"]; ok {
// if openvswitch do not remove let's do the job by ovs piece of code
if driver == "openvswitch" {
u.Graph.Unlink(u.Root, intf)
} else {
u.Graph.DelNode(intf)
}
}
}
delete(u.indexToChildrenQueue, int64(index))
}
func matchRoute(ifaceName string, dest net.IP) func(m syscall.NetlinkMessage) (bool, error) {
return func(m syscall.NetlinkMessage) (bool, error) {
switch m.Header.Type {
case syscall.RTM_NEWROUTE:
route, err := deserializeRoute(m.Data)
if err != nil {
return true, err
}
link, _ := netlink.LinkByIndex(route.LinkIndex)
if link.Attrs().Name == ifaceName && route.Dst.IP.Equal(dest) {
return true, nil
}
}
return false, nil
}
}
func resourceLXCBridgeRead(d *schema.ResourceData, meta interface{}) error {
bridgeIndex, err := strconv.Atoi(d.Id())
if err != nil {
return fmt.Errorf("Internal error reading resource ID: %v", err)
}
bridge, err := netlink.LinkByIndex(bridgeIndex)
if err != nil {
return fmt.Errorf("Unable to find bridge %v: %v", bridgeIndex, err)
}
d.Set("mac", bridge.Attrs().HardwareAddr.String())
log.Printf("[INFO] Bridge info: %v", bridge)
return nil
}
func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(netlink.Route) error) error {
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
if err != nil {
return err
}
for _, route := range routes {
if link, err := netlink.LinkByIndex(route.LinkIndex); err == nil {
if _, found := ignoreIfaceNames[link.Attrs().Name]; found {
continue
}
}
if err := check(route); err != nil {
return err
}
}
return nil
}
// Adds a macvlan interface to a container for use with the egress router feature
func addMacvlan(netns string) error {
var defIface netlink.Link
var err error
// Find interface with the default route
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
if err != nil {
return fmt.Errorf("failed to read routes: %v", err)
}
for _, r := range routes {
if r.Dst == nil {
defIface, err = netlink.LinkByIndex(r.LinkIndex)
if err != nil {
return fmt.Errorf("failed to get default route interface: %v", err)
}
}
}
if defIface == nil {
return fmt.Errorf("failed to find default route interface")
}
return ns.WithNetNSPath(netns, func(ns.NetNS) error {
err := netlink.LinkAdd(&netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
MTU: defIface.Attrs().MTU,
Name: "macvlan0",
ParentIndex: defIface.Attrs().Index,
},
Mode: netlink.MACVLAN_MODE_PRIVATE,
})
if err != nil {
return fmt.Errorf("failed to create macvlan interface: %v", err)
}
l, err := netlink.LinkByName("macvlan0")
if err != nil {
return fmt.Errorf("failed to find macvlan interface: %v", err)
}
err = netlink.LinkSetUp(l)
if err != nil {
return fmt.Errorf("failed to set macvlan interface up: %v", err)
}
return nil
})
}
func testAccCheckLXCBridgeDestroy(s *terraform.State) error {
for _, rs := range s.RootModule().Resources {
if rs.Type != "lxc_bridge" {
continue
}
bridgeIndex, err := strconv.Atoi(rs.Primary.ID)
if err != nil {
return fmt.Errorf("Internal error reading resource ID.")
}
_, err = netlink.LinkByIndex(bridgeIndex)
if err == nil {
return fmt.Errorf("Bridge still exists.")
}
}
return nil
}
// For a given container, returns host veth name, container veth MAC, and pod IP
func getVethInfo(netns, containerIfname string) (string, string, string, error) {
var (
peerIfindex int
contVeth netlink.Link
err error
podIP string
)
containerNs, err := ns.GetNS(netns)
if err != nil {
return "", "", "", fmt.Errorf("failed to get container netns: %v", err)
}
defer containerNs.Close()
err = containerNs.Do(func(ns.NetNS) error {
contVeth, err = netlink.LinkByName(containerIfname)
if err != nil {
return err
}
peerIfindex = contVeth.Attrs().ParentIndex
addrs, err := netlink.AddrList(contVeth, syscall.AF_INET)
if err != nil {
return fmt.Errorf("failed to get container IP addresses: %v", err)
}
if len(addrs) == 0 {
return fmt.Errorf("container had no addresses")
}
podIP = addrs[0].IP.String()
return nil
})
if err != nil {
return "", "", "", fmt.Errorf("failed to inspect container interface: %v", err)
}
hostVeth, err := netlink.LinkByIndex(peerIfindex)
if err != nil {
return "", "", "", fmt.Errorf("failed to get host veth: %v", err)
}
return hostVeth.Attrs().Name, contVeth.Attrs().HardwareAddr.String(), podIP, nil
}
func ensureLink(vxlan *netlink.Vxlan) (*netlink.Vxlan, error) {
err := netlink.LinkAdd(vxlan)
if err == syscall.EEXIST {
// it's ok if the device already exists as long as config is similar
existing, err := netlink.LinkByName(vxlan.Name)
if err != nil {
return nil, err
}
incompat := vxlanLinksIncompat(vxlan, existing)
if incompat == "" {
return existing.(*netlink.Vxlan), nil
}
// delete existing
log.Warningf("%q already exists with incompatable configuration: %v; recreating device", vxlan.Name, incompat)
if err = netlink.LinkDel(existing); err != nil {
return nil, fmt.Errorf("failed to delete interface: %v", err)
}
// create new
if err = netlink.LinkAdd(vxlan); err != nil {
return nil, fmt.Errorf("failed to create vxlan interface: %v", err)
}
} else if err != nil {
return nil, err
}
ifindex := vxlan.Index
link, err := netlink.LinkByIndex(vxlan.Index)
if err != nil {
return nil, fmt.Errorf("can't locate created vxlan device with index %v", ifindex)
}
var ok bool
if vxlan, ok = link.(*netlink.Vxlan); !ok {
return nil, fmt.Errorf("created vxlan device with index %v is not vxlan", ifindex)
}
return vxlan, nil
}
func (mapper *NetLinkMapper) cacheUpdater() {
logging.GetLogger().Debug("Start NetLink cache updater")
var ifIndex uint32
for {
ifIndex = <-mapper.cacheUpdaterChan
logging.GetLogger().Debug("ifIndex request received: %s", ifIndex)
var attrs *netlink.LinkAttrs
link, err := netlink.LinkByIndex(int(ifIndex))
if err != nil {
logging.GetLogger().Error("Error while getting interface by index via netling: ", err.Error())
attrs = &netlink.LinkAttrs{}
} else {
attrs = link.Attrs()
}
mapper.cache.Set(strconv.Itoa(int(ifIndex)), attrs, cache.DefaultExpiration)
}
}
// Wait for an interface to come up and have a route added to the multicast subnet.
// This matches the behaviour in 'weave attach', which is the only context in which
// we expect this to be called. If you change one, change the other to match.
func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) {
iface, err := ensureInterface(ifaceName)
if err != nil {
return nil, err
}
dest := net.IPv4(224, 0, 0, 0)
if CheckRouteExists(ifaceName, dest) {
return iface, err
}
ch := make(chan netlink.RouteUpdate)
done := make(chan struct{})
defer close(done)
if err := netlink.RouteSubscribe(ch, done); err != nil {
return nil, err
}
for update := range ch {
link, _ := netlink.LinkByIndex(update.Route.LinkIndex)
if link.Attrs().Name == ifaceName && update.Route.Dst.IP.Equal(dest) {
break
}
}
return iface, nil
}
func (nm *NetlinkMonitor) handleMasterChange(node *ExtInterface, link netlink.Link, isAdd bool) error {
newMasterIdx := link.Attrs().MasterIndex
Debug.Printf("link %s master %d\n", node.Link().Attrs().Name, newMasterIdx)
if newMasterIdx != node.Link().Attrs().MasterIndex || isAdd {
if newMasterIdx != 0 {
// add case
masterLink, err := netlink.LinkByIndex(newMasterIdx)
if err != nil {
return err
}
bridge, ok := masterLink.(*netlink.Bridge)
if !ok {
return fmt.Errorf("unsupported non-bridge master")
}
master := nm.ensureBridge(bridge)
if node.ID() < 0 {
node.SetID(nm.g.NewNodeID())
nm.g.AddNode(node)
}
// set to 0 so that the normal egress path is taken
fid, tid := 0, 0
nm.g.SetEdge(canvas.NewEdgeChain(node, master, &fid, &tid))
nm.g.SetEdge(canvas.NewEdgeChain(master, node, &tid, &fid))
if err := nm.r.Provision(nm.g, []InterfaceNode{node}); err != nil {
return err
}
if err := nm.ensureInterface(nm.g, node); err != nil {
return err
}
nm.r.Run(nm.g, []InterfaceNode{node})
} else {
// remove case
}
}
return nil
}
func (*nl) LinkByIndex(index int) (netlink.Link, error) {
return netlink.LinkByIndex(index)
}
// Set up all networking (host/container veth, OVS flows, IPAM, loopback, etc)
func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubehostport.RunningPod, error) {
podConfig, pod, err := m.getPodConfig(req)
if err != nil {
return nil, nil, err
}
ipamResult, err := m.ipamAdd(req.Netns, req.ContainerId)
if err != nil {
// TODO: Remove this hack once we've figured out how to retrieve the netns
// of an exited container. Currently, restarting docker will leak a bunch of
// ips. This will exhaust available ip space unless we cleanup old ips. At the
// same time we don't want to try GC'ing them periodically as that could lead
// to a performance regression in starting pods. So on each setup failure, try
// GC on the assumption that the kubelet is going to retry pod creation, and
// when it does, there will be ips.
m.ipamGarbageCollection()
return nil, nil, fmt.Errorf("failed to run IPAM for %v: %v", req.ContainerId, err)
}
podIP := ipamResult.IP4.IP.IP
// Release any IPAM allocations and hostports if the setup failed
var success bool
defer func() {
if !success {
m.ipamDel(req.ContainerId)
if err := m.hostportHandler.SyncHostports(TUN, m.getRunningPods()); err != nil {
glog.Warningf("failed syncing hostports: %v", err)
}
}
}()
// Open any hostports the pod wants
newPod := &kubehostport.RunningPod{Pod: pod, IP: podIP}
if err := m.hostportHandler.OpenPodHostportsAndSync(newPod, TUN, m.getRunningPods()); err != nil {
return nil, nil, err
}
var hostVeth, contVeth netlink.Link
err = ns.WithNetNSPath(req.Netns, func(hostNS ns.NetNS) error {
hostVeth, contVeth, err = ip.SetupVeth(podInterfaceName, int(m.mtu), hostNS)
if err != nil {
return fmt.Errorf("failed to create container veth: %v", err)
}
// refetch to get hardware address and other properties
contVeth, err = netlink.LinkByIndex(contVeth.Attrs().Index)
if err != nil {
return fmt.Errorf("failed to fetch container veth: %v", err)
}
// Clear out gateway to prevent ConfigureIface from adding the cluster
// subnet via the gateway
ipamResult.IP4.Gateway = nil
if err = ipam.ConfigureIface(podInterfaceName, ipamResult); err != nil {
return fmt.Errorf("failed to configure container IPAM: %v", err)
}
lo, err := netlink.LinkByName("lo")
if err == nil {
err = netlink.LinkSetUp(lo)
}
if err != nil {
return fmt.Errorf("failed to configure container loopback: %v", err)
}
return nil
})
if err != nil {
return nil, nil, err
}
if podConfig.wantMacvlan {
if err := addMacvlan(req.Netns); err != nil {
return nil, nil, err
}
}
contVethMac := contVeth.Attrs().HardwareAddr.String()
vnidStr := vnidToString(podConfig.vnid)
out, err := exec.Command(sdnScript, setUpCmd, hostVeth.Attrs().Name, contVethMac, podIP.String(), vnidStr, podConfig.ingressBandwidth, podConfig.egressBandwidth).CombinedOutput()
glog.V(5).Infof("SetUpPod network plugin output: %s, %v", string(out), err)
if isScriptError(err) {
return nil, nil, fmt.Errorf("error running network setup script:\nhostVethName %s, contVethMac %s, podIP %s, podConfig %#v\n %s", hostVeth.Attrs().Name, contVethMac, podIP.String(), podConfig, getScriptError(out))
} else if err != nil {
return nil, nil, err
}
success = true
return ipamResult, newPod, nil
}
// Set up all networking (host/container veth, OVS flows, IPAM, loopback, etc)
func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubehostport.RunningPod, error) {
podConfig, pod, err := m.getPodConfig(req)
if err != nil {
return nil, nil, err
}
ipamResult, err := m.runIPAM(req.Netns, cniserver.CNI_ADD, req.ContainerId)
if err != nil {
return nil, nil, fmt.Errorf("failed to run IPAM for %v: %v", req.ContainerId, err)
}
podIP := ipamResult.IP4.IP.IP
// Release any IPAM allocations and hostports if the setup failed
var success bool
defer func() {
if !success {
m.runIPAM(req.Netns, cniserver.CNI_DEL, req.ContainerId)
if err := m.hostportHandler.SyncHostports(TUN, m.getRunningPods()); err != nil {
glog.Warningf("failed syncing hostports: %v", err)
}
}
}()
// Open any hostports the pod wants
newPod := &kubehostport.RunningPod{Pod: pod, IP: podIP}
if err := m.hostportHandler.OpenPodHostportsAndSync(newPod, TUN, m.getRunningPods()); err != nil {
return nil, nil, err
}
var hostVeth, contVeth netlink.Link
err = ns.WithNetNSPath(req.Netns, func(hostNS ns.NetNS) error {
hostVeth, contVeth, err = ip.SetupVeth(podInterfaceName, int(m.mtu), hostNS)
if err != nil {
return fmt.Errorf("failed to create container veth: %v", err)
}
// refetch to get hardware address and other properties
contVeth, err = netlink.LinkByIndex(contVeth.Attrs().Index)
if err != nil {
return fmt.Errorf("failed to fetch container veth: %v", err)
}
// Clear out gateway to prevent ConfigureIface from adding the cluster
// subnet via the gateway
ipamResult.IP4.Gateway = nil
if err = ipam.ConfigureIface(podInterfaceName, ipamResult); err != nil {
return fmt.Errorf("failed to configure container IPAM: %v", err)
}
lo, err := netlink.LinkByName("lo")
if err == nil {
err = netlink.LinkSetUp(lo)
}
if err != nil {
return fmt.Errorf("failed to configure container loopback: %v", err)
}
return nil
})
if err != nil {
return nil, nil, err
}
if podConfig.wantMacvlan {
if err := addMacvlan(req.Netns); err != nil {
return nil, nil, err
}
}
contVethMac := contVeth.Attrs().HardwareAddr.String()
vnidStr := vnidToString(podConfig.vnid)
out, err := exec.Command(sdnScript, setUpCmd, hostVeth.Attrs().Name, contVethMac, podIP.String(), vnidStr, podConfig.ingressBandwidth, podConfig.egressBandwidth).CombinedOutput()
glog.V(5).Infof("SetUpPod network plugin output: %s, %v", string(out), err)
if isScriptError(err) {
return nil, nil, fmt.Errorf("error running network setup script:\nhostVethName %s, contVethMac %s, podIP %s, podConfig %#v\n %s", hostVeth.Attrs().Name, contVethMac, podIP.String(), podConfig, getScriptError(out))
} else if err != nil {
return nil, nil, err
}
success = true
return ipamResult, newPod, nil
}