// delVlanLink verifies only sub-interfaces with a vlan id get deleted
func delVlanLink(linkName string) error {
if strings.Contains(linkName, ".") {
_, _, err := parseVlan(linkName)
if err != nil {
return err
}
// delete the vlan subinterface
vlanLink, err := ns.NlHandle().LinkByName(linkName)
if err != nil {
return fmt.Errorf("failed to find interface %s on the Docker host : %v", linkName, err)
}
// verify a parent interface isn't being deleted
if vlanLink.Attrs().ParentIndex == 0 {
return fmt.Errorf("interface %s does not appear to be a slave device: %v", linkName, err)
}
// delete the macvlan slave device
if err := ns.NlHandle().LinkDel(vlanLink); err != nil {
return fmt.Errorf("failed to delete %s link: %v", linkName, err)
}
logrus.Debugf("Deleted a vlan tagged netlink subinterface: %s", linkName)
}
// if the subinterface doesn't parse to iface.vlan_id leave the interface in
// place since it could be a user specified name not created by the driver.
return nil
}
// DeleteEndpoint remove the endpoint and associated netlink interface
func (d *driver) DeleteEndpoint(nid, eid string) error {
defer osl.InitOSContext()()
if err := validateID(nid, eid); err != nil {
return err
}
n := d.network(nid)
if n == nil {
return fmt.Errorf("network id %q not found", nid)
}
ep := n.endpoint(eid)
if ep == nil {
return fmt.Errorf("endpoint id %q not found", eid)
}
if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
ns.NlHandle().LinkDel(link)
}
if err := d.storeDelete(ep); err != nil {
logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err)
}
n.deleteEndpoint(ep.id)
return nil
}
// Create the macvlan slave specifying the source name
func createMacVlan(containerIfName, parent, macvlanMode string) (string, error) {
// Set the macvlan mode. Default is bridge mode
mode, err := setMacVlanMode(macvlanMode)
if err != nil {
return "", fmt.Errorf("Unsupported %s macvlan mode: %v", macvlanMode, err)
}
// verify the Docker host interface acting as the macvlan parent iface exists
if !parentExists(parent) {
return "", fmt.Errorf("the requested parent interface %s was not found on the Docker host", parent)
}
// Get the link for the master index (Example: the docker host eth iface)
parentLink, err := ns.NlHandle().LinkByName(parent)
if err != nil {
return "", fmt.Errorf("error occoured looking up the %s parent iface %s error: %s", macvlanType, parent, err)
}
// Create a macvlan link
macvlan := &netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
Name: containerIfName,
ParentIndex: parentLink.Attrs().Index,
},
Mode: mode,
}
if err := ns.NlHandle().LinkAdd(macvlan); err != nil {
// If a user creates a macvlan and ipvlan on same parent, only one slave iface can be active at a time.
return "", fmt.Errorf("failed to create the %s port: %v", macvlanType, err)
}
return macvlan.Attrs().Name, nil
}
func deleteInterface(name string) error {
defer osl.InitOSContext()()
link, err := ns.NlHandle().LinkByName(name)
if err != nil {
return fmt.Errorf("failed to find interface with name %s: %v", name, err)
}
if err := ns.NlHandle().LinkDel(link); err != nil {
return fmt.Errorf("error deleting interface with name %s: %v", name, err)
}
return nil
}
func TestAddRemoveInterface(t *testing.T) {
defer testutils.SetupTestOSContext(t)()
key, err := newKey(t)
if err != nil {
t.Fatalf("Failed to obtain a key: %v", err)
}
s, err := NewSandbox(key, true, false)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
runtime.LockOSThread()
if s.Key() != key {
t.Fatalf("s.Key() returned %s. Expected %s", s.Key(), key)
}
tbox, err := newInfo(ns.NlHandle(), t)
if err != nil {
t.Fatalf("Failed to generate new sandbox info: %v", err)
}
for _, i := range tbox.Info().Interfaces() {
err = s.AddInterface(i.SrcName(), i.DstName(),
tbox.InterfaceOptions().Bridge(i.Bridge()),
tbox.InterfaceOptions().Address(i.Address()),
tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6()))
if err != nil {
t.Fatalf("Failed to add interfaces to sandbox: %v", err)
}
}
verifySandbox(t, s, []string{"0", "1", "2"})
interfaces := s.Info().Interfaces()
if err := interfaces[0].Remove(); err != nil {
t.Fatalf("Failed to remove interfaces from sandbox: %v", err)
}
verifySandbox(t, s, []string{"1", "2"})
i := tbox.Info().Interfaces()[0]
if err := s.AddInterface(i.SrcName(), i.DstName(),
tbox.InterfaceOptions().Bridge(i.Bridge()),
tbox.InterfaceOptions().Address(i.Address()),
tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())); err != nil {
t.Fatalf("Failed to add interfaces to sandbox: %v", err)
}
verifySandbox(t, s, []string{"1", "2", "3"})
err = s.Destroy()
if err != nil {
t.Fatal(err)
}
GC()
verifyCleanup(t, s, false)
}
func deleteInterfaceBySubnet(brPrefix string, s *subnet) error {
defer osl.InitOSContext()()
nlh := ns.NlHandle()
links, err := nlh.LinkList()
if err != nil {
return fmt.Errorf("failed to list interfaces while deleting bridge interface by subnet: %v", err)
}
for _, l := range links {
name := l.Attrs().Name
if _, ok := l.(*netlink.Bridge); ok && strings.HasPrefix(name, brPrefix) {
addrList, err := nlh.AddrList(l, netlink.FAMILY_V4)
if err != nil {
logrus.Errorf("error getting AddressList for bridge %s", name)
continue
}
for _, addr := range addrList {
if netutils.NetworkOverlaps(addr.IPNet, s.subnetIP) {
err = nlh.LinkDel(l)
if err != nil {
logrus.Errorf("error deleting bridge (%s) with subnet %v: %v", name, addr.IPNet, err)
}
}
}
}
}
return nil
}
func (d *driver) DeleteEndpoint(nid, eid string) error {
nlh := ns.NlHandle()
if err := validateID(nid, eid); err != nil {
return err
}
n := d.network(nid)
if n == nil {
return fmt.Errorf("network id %q not found", nid)
}
ep := n.endpoint(eid)
if ep == nil {
return fmt.Errorf("endpoint id %q not found", eid)
}
n.deleteEndpoint(eid)
if ep.ifName == "" {
return nil
}
link, err := nlh.LinkByName(ep.ifName)
if err != nil {
log.Debugf("Failed to retrieve interface (%s)'s link on endpoint (%s) delete: %v", ep.ifName, ep.id, err)
return nil
}
if err := nlh.LinkDel(link); err != nil {
log.Debugf("Failed to delete interface (%s)'s link on endpoint (%s) delete: %v", ep.ifName, ep.id, err)
}
return nil
}
func deleteVxlanByVNI(path string, vni uint32) error {
defer osl.InitOSContext()()
nlh := ns.NlHandle()
if path != "" {
ns, err := netns.GetFromPath(path)
if err != nil {
return fmt.Errorf("failed to get ns handle for %s: %v", path, err)
}
defer ns.Close()
nlh, err = netlink.NewHandleAt(ns)
if err != nil {
return fmt.Errorf("failed to get netlink handle for ns %s: %v", path, err)
}
defer nlh.Delete()
}
links, err := nlh.LinkList()
if err != nil {
return fmt.Errorf("failed to list interfaces while deleting vxlan interface by vni: %v", err)
}
for _, l := range links {
if l.Type() == "vxlan" && (vni == 0 || l.(*netlink.Vxlan).VxlanId == int(vni)) {
err = nlh.LinkDel(l)
if err != nil {
return fmt.Errorf("error deleting vxlan interface with id %d: %v", vni, err)
}
return nil
}
}
return fmt.Errorf("could not find a vxlan interface to delete with id %d", vni)
}
func createVethPair() (string, string, error) {
defer osl.InitOSContext()()
nlh := ns.NlHandle()
// Generate a name for what will be the host side pipe interface
name1, err := netutils.GenerateIfaceName(nlh, vethPrefix, vethLen)
if err != nil {
return "", "", fmt.Errorf("error generating veth name1: %v", err)
}
// Generate a name for what will be the sandbox side pipe interface
name2, err := netutils.GenerateIfaceName(nlh, vethPrefix, vethLen)
if err != nil {
return "", "", fmt.Errorf("error generating veth name2: %v", err)
}
// Generate and add the interface pipe host <-> sandbox
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: name1, TxQLen: 0},
PeerName: name2}
if err := nlh.LinkAdd(veth); err != nil {
return "", "", fmt.Errorf("error creating veth pair: %v", err)
}
return name1, name2, nil
}
// parentExists check if the specified interface exists in the default namespace
func parentExists(ifaceStr string) bool {
_, err := ns.NlHandle().LinkByName(ifaceStr)
if err != nil {
return false
}
return true
}
// DeleteNetwork the network for the specified driver type
func (d *driver) DeleteNetwork(nid string) error {
defer osl.InitOSContext()()
n := d.network(nid)
if n == nil {
return fmt.Errorf("network id %s not found", nid)
}
// if the driver created the slave interface, delete it, otherwise leave it
if ok := n.config.CreatedSlaveLink; ok {
// if the interface exists, only delete if it matches iface.vlan or dummy.net_id naming
if ok := parentExists(n.config.Parent); ok {
// only delete the link if it is named the net_id
if n.config.Parent == getDummyName(stringid.TruncateID(nid)) {
err := delDummyLink(n.config.Parent)
if err != nil {
logrus.Debugf("link %s was not deleted, continuing the delete network operation: %v",
n.config.Parent, err)
}
} else {
// only delete the link if it matches iface.vlan naming
err := delVlanLink(n.config.Parent)
if err != nil {
logrus.Debugf("link %s was not deleted, continuing the delete network operation: %v",
n.config.Parent, err)
}
}
}
}
for _, ep := range n.endpoints {
if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
ns.NlHandle().LinkDel(link)
}
if err := d.storeDelete(ep); err != nil {
logrus.Warnf("Failed to remove macvlan endpoint %s from store: %v", ep.id[0:7], err)
}
}
// delete the *network
d.deleteNetwork(nid)
// delete the network record from persistent cache
err := d.storeDelete(n.config)
if err != nil {
return fmt.Errorf("error deleting deleting id %s from datastore: %v", nid, err)
}
return nil
}
// delDummyLink deletes the link type dummy used when -o parent is not passed
func delDummyLink(linkName string) error {
// delete the vlan subinterface
dummyLink, err := ns.NlHandle().LinkByName(linkName)
if err != nil {
return fmt.Errorf("failed to find link %s on the Docker host : %v", linkName, err)
}
// verify a parent interface is being deleted
if dummyLink.Attrs().ParentIndex != 0 {
return fmt.Errorf("link %s is not a parent dummy interface", linkName)
}
// delete the macvlan dummy device
if err := ns.NlHandle().LinkDel(dummyLink); err != nil {
return fmt.Errorf("failed to delete the dummy %s link: %v", linkName, err)
}
logrus.Debugf("Deleted a dummy parent link: %s", linkName)
return nil
}
func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
action := "Removing"
xfrmProgram := ns.NlHandle().XfrmPolicyDel
if add {
action = "Adding"
xfrmProgram = ns.NlHandle().XfrmPolicyAdd
}
fullMask := net.CIDRMask(8*len(fSA.Src), 8*len(fSA.Src))
fPol := &netlink.XfrmPolicy{
Src: &net.IPNet{IP: fSA.Src, Mask: fullMask},
Dst: &net.IPNet{IP: fSA.Dst, Mask: fullMask},
Dir: netlink.XFRM_DIR_OUT,
Proto: 17,
DstPort: 4789,
Mark: &netlink.XfrmMark{
Value: mark,
},
Tmpls: []netlink.XfrmPolicyTmpl{
{
Src: fSA.Src,
Dst: fSA.Dst,
Proto: netlink.XFRM_PROTO_ESP,
Mode: netlink.XFRM_MODE_TRANSPORT,
Spi: fSA.Spi,
},
},
}
exists, err := spExists(fPol)
if err != nil {
exists = !add
}
if add != exists {
logrus.Debugf("%s fSP{%s}", action, fPol)
if err := xfrmProgram(fPol); err != nil {
logrus.Warnf("%s fSP{%s}: %v", action, fPol, err)
}
}
return nil
}
// DeleteEndpoint remove the endpoint and associated netlink interface
func (d *driver) DeleteEndpoint(nid, eid string) error {
defer osl.InitOSContext()()
if err := validateID(nid, eid); err != nil {
return err
}
n := d.network(nid)
if n == nil {
return fmt.Errorf("network id %q not found", nid)
}
ep := n.endpoint(eid)
if ep == nil {
return fmt.Errorf("endpoint id %q not found", eid)
}
if link, err := ns.NlHandle().LinkByName(ep.srcName); err == nil {
ns.NlHandle().LinkDel(link)
}
return nil
}
func (d *driver) DeleteNetwork(nid string) error {
if nid == "" {
return fmt.Errorf("invalid network id")
}
// Make sure driver resources are initialized before proceeding
if err := d.configure(); err != nil {
return err
}
n := d.network(nid)
if n == nil {
return fmt.Errorf("could not find network with id %s", nid)
}
for _, ep := range n.endpoints {
if ep.ifName != "" {
if link, err := ns.NlHandle().LinkByName(ep.ifName); err != nil {
ns.NlHandle().LinkDel(link)
}
}
if err := d.deleteEndpointFromStore(ep); err != nil {
logrus.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)
}
}
d.deleteNetwork(nid)
vnis, err := n.releaseVxlanID()
if err != nil {
return err
}
if n.secure {
for _, vni := range vnis {
programMangle(vni, false)
}
}
return nil
}
// createDummyLink creates a dummy0 parent link
func createDummyLink(dummyName, truncNetID string) error {
// create a parent interface since one was not specified
parent := &netlink.Dummy{
LinkAttrs: netlink.LinkAttrs{
Name: dummyName,
},
}
if err := ns.NlHandle().LinkAdd(parent); err != nil {
return err
}
parentDummyLink, err := ns.NlHandle().LinkByName(dummyName)
if err != nil {
return fmt.Errorf("error occoured looking up the %s parent iface %s error: %s", macvlanType, dummyName, err)
}
// bring the new netlink iface up
if err := ns.NlHandle().LinkSetUp(parentDummyLink); err != nil {
return fmt.Errorf("failed to enable %s the macvlan parent link: %v", dummyName, err)
}
return nil
}
func spExists(sp *netlink.XfrmPolicy) (bool, error) {
_, err := ns.NlHandle().XfrmPolicyGet(sp)
switch err {
case nil:
return true, nil
case syscall.ENOENT:
return false, nil
default:
err = fmt.Errorf("Error while checking for SP existence: %v", err)
logrus.Warn(err)
return false, err
}
}
func saExists(sa *netlink.XfrmState) (bool, error) {
_, err := ns.NlHandle().XfrmStateGet(sa)
switch err {
case nil:
return true, nil
case syscall.ESRCH:
return false, nil
default:
err = fmt.Errorf("Error while checking for SA existence: %v", err)
logrus.Warn(err)
return false, err
}
}
// ElectInterfaceAddresses looks for an interface on the OS with the
// specified name and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
// If the interface does not exist, it chooses from a predefined
// list the first IPv4 address which does not conflict with other
// interfaces on the system.
func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
var (
v4Nets []*net.IPNet
v6Nets []*net.IPNet
)
defer osl.InitOSContext()()
link, _ := ns.NlHandle().LinkByName(name)
if link != nil {
v4addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V4)
if err != nil {
return nil, nil, err
}
v6addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V6)
if err != nil {
return nil, nil, err
}
for _, nlAddr := range v4addr {
v4Nets = append(v4Nets, nlAddr.IPNet)
}
for _, nlAddr := range v6addr {
v6Nets = append(v6Nets, nlAddr.IPNet)
}
}
if link == nil || len(v4Nets) == 0 {
// Choose from predefined broad networks
v4Net, err := FindAvailableNetwork(ipamutils.PredefinedBroadNetworks)
if err != nil {
return nil, nil, err
}
v4Nets = append(v4Nets, v4Net)
}
return v4Nets, v6Nets, nil
}
func bridgeInterfaceExists(name string) (bool, error) {
nlh := ns.NlHandle()
link, err := nlh.LinkByName(name)
if err != nil {
if strings.Contains(err.Error(), "Link not found") {
return false, nil
}
return false, fmt.Errorf("failed to check bridge interface existence: %v", err)
}
if link.Type() == "bridge" {
return true, nil
}
return false, fmt.Errorf("existing interface %s is not a bridge", name)
}
// createVlanLink parses sub-interfaces and vlan id for creation
func createVlanLink(parentName string) error {
if strings.Contains(parentName, ".") {
parent, vidInt, err := parseVlan(parentName)
if err != nil {
return err
}
// VLAN identifier or VID is a 12-bit field specifying the VLAN to which the frame belongs
if vidInt > 4094 || vidInt < 1 {
return fmt.Errorf("vlan id must be between 1-4094, received: %d", vidInt)
}
// get the parent link to attach a vlan subinterface
parentLink, err := ns.NlHandle().LinkByName(parent)
if err != nil {
return fmt.Errorf("failed to find master interface %s on the Docker host: %v", parent, err)
}
vlanLink := &netlink.Vlan{
LinkAttrs: netlink.LinkAttrs{
Name: parentName,
ParentIndex: parentLink.Attrs().Index,
},
VlanId: vidInt,
}
// create the subinterface
if err := ns.NlHandle().LinkAdd(vlanLink); err != nil {
return fmt.Errorf("failed to create %s vlan link: %v", vlanLink.Name, err)
}
// Bring the new netlink iface up
if err := ns.NlHandle().LinkSetUp(vlanLink); err != nil {
return fmt.Errorf("failed to enable %s the macvlan parent link %v", vlanLink.Name, err)
}
logrus.Debugf("Added a vlan tagged netlink subinterface: %s with a vlan id: %d", parentName, vidInt)
return nil
}
return fmt.Errorf("invalid subinterface vlan name %s, example formatting is eth0.10", parentName)
}
// CheckRouteOverlaps checks whether the passed network overlaps with any existing routes
func CheckRouteOverlaps(toCheck *net.IPNet) error {
if networkGetRoutesFct == nil {
networkGetRoutesFct = ns.NlHandle().RouteList
}
networks, err := networkGetRoutesFct(nil, netlink.FAMILY_V4)
if err != nil {
return err
}
for _, network := range networks {
if network.Dst != nil && NetworkOverlaps(toCheck, network.Dst) {
return ErrNetworkOverlaps
}
}
return nil
}
func createVxlan(name string, vni uint32) error {
defer osl.InitOSContext()()
vxlan := &netlink.Vxlan{
LinkAttrs: netlink.LinkAttrs{Name: name},
VxlanId: int(vni),
Learning: true,
Port: vxlanPort,
Proxy: true,
L3miss: true,
L2miss: true,
}
if err := ns.NlHandle().LinkAdd(vxlan); err != nil {
return fmt.Errorf("error creating vxlan interface: %v", err)
}
return nil
}
func findOIFName(ip net.IP) (string, error) {
nlh := ns.NlHandle()
routes, err := nlh.RouteGet(ip)
if err != nil {
return "", err
}
if len(routes) == 0 {
return "", fmt.Errorf("no route to %s", ip)
}
// Pick the first route(typically there is only one route). We
// don't support multipath.
link, err := nlh.LinkByIndex(routes[0].LinkIndex)
if err != nil {
return "", err
}
return link.Attrs().Name, nil
}
func networkOnceInit() {
populateVNITbl()
if os.Getenv("_OVERLAY_HOST_MODE") != "" {
hostMode = true
return
}
err := createVxlan("testvxlan", 1)
if err != nil {
logrus.Errorf("Failed to create testvxlan interface: %v", err)
return
}
defer deleteInterface("testvxlan")
path := "/proc/self/ns/net"
hNs, err := netns.GetFromPath(path)
if err != nil {
logrus.Errorf("Failed to get network namespace from path %s while setting host mode: %v", path, err)
return
}
defer hNs.Close()
nlh := ns.NlHandle()
iface, err := nlh.LinkByName("testvxlan")
if err != nil {
logrus.Errorf("Failed to get link testvxlan while setting host mode: %v", err)
return
}
// If we are not able to move the vxlan interface to a namespace
// then fallback to host mode
if err := nlh.LinkSetNsFd(iface, int(hNs)); err != nil {
hostMode = true
}
}
func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (fSA *netlink.XfrmState, rSA *netlink.XfrmState, err error) {
var (
action = "Removing"
xfrmProgram = ns.NlHandle().XfrmStateDel
)
if add {
action = "Adding"
xfrmProgram = ns.NlHandle().XfrmStateAdd
}
if dir&reverse > 0 {
rSA = &netlink.XfrmState{
Src: remoteIP,
Dst: localIP,
Proto: netlink.XFRM_PROTO_ESP,
Spi: spi.reverse,
Mode: netlink.XFRM_MODE_TRANSPORT,
}
if add {
rSA.Aead = buildAeadAlgo(k, spi.reverse)
}
exists, err := saExists(rSA)
if err != nil {
exists = !add
}
if add != exists {
logrus.Debugf("%s: rSA{%s}", action, rSA)
if err := xfrmProgram(rSA); err != nil {
logrus.Warnf("Failed %s rSA{%s}: %v", action, rSA, err)
}
}
}
if dir&forward > 0 {
fSA = &netlink.XfrmState{
Src: localIP,
Dst: remoteIP,
Proto: netlink.XFRM_PROTO_ESP,
Spi: spi.forward,
Mode: netlink.XFRM_MODE_TRANSPORT,
}
if add {
fSA.Aead = buildAeadAlgo(k, spi.forward)
}
exists, err := saExists(fSA)
if err != nil {
exists = !add
}
if add != exists {
logrus.Debugf("%s fSA{%s}", action, fSA)
if err := xfrmProgram(fSA); err != nil {
logrus.Warnf("Failed %s fSA{%s}: %v.", action, fSA, err)
}
}
}
return
}
// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
if err := validateID(nid, eid); err != nil {
return err
}
n := d.network(nid)
if n == nil {
return fmt.Errorf("could not find network with id %s", nid)
}
ep := n.endpoint(eid)
if ep == nil {
return fmt.Errorf("could not find endpoint with id %s", eid)
}
if n.secure && len(d.keys) == 0 {
return fmt.Errorf("cannot join secure network: encryption keys not present")
}
nlh := ns.NlHandle()
if n.secure && !nlh.SupportsNetlinkFamily(syscall.NETLINK_XFRM) {
return fmt.Errorf("cannot join secure network: required modules to install IPSEC rules are missing on host")
}
s := n.getSubnetforIP(ep.addr)
if s == nil {
return fmt.Errorf("could not find subnet for endpoint %s", eid)
}
if err := n.obtainVxlanID(s); err != nil {
return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
}
if err := n.joinSandbox(false); err != nil {
return fmt.Errorf("network sandbox join failed: %v", err)
}
if err := n.joinSubnetSandbox(s, false); err != nil {
return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
}
// joinSubnetSandbox gets called when an endpoint comes up on a new subnet in the
// overlay network. Hence the Endpoint count should be updated outside joinSubnetSandbox
n.incEndpointCount()
sbox := n.sandbox()
overlayIfName, containerIfName, err := createVethPair()
if err != nil {
return err
}
ep.ifName = containerIfName
if err := d.writeEndpointToStore(ep); err != nil {
return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err)
}
// Set the container interface and its peer MTU to 1450 to allow
// for 50 bytes vxlan encap (inner eth header(14) + outer IP(20) +
// outer UDP(8) + vxlan header(8))
mtu := n.maxMTU()
veth, err := nlh.LinkByName(overlayIfName)
if err != nil {
return fmt.Errorf("cound not find link by name %s: %v", overlayIfName, err)
}
err = nlh.LinkSetMTU(veth, mtu)
if err != nil {
return err
}
if err := sbox.AddInterface(overlayIfName, "veth",
sbox.InterfaceOptions().Master(s.brName)); err != nil {
return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
}
veth, err = nlh.LinkByName(containerIfName)
if err != nil {
return fmt.Errorf("could not find link by name %s: %v", containerIfName, err)
}
err = nlh.LinkSetMTU(veth, mtu)
if err != nil {
return err
}
if err := nlh.LinkSetHardwareAddr(veth, ep.mac); err != nil {
return fmt.Errorf("could not set mac address (%v) to the container interface: %v", ep.mac, err)
}
for _, sub := range n.subnets {
if sub == s {
continue
}
if err := jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil {
log.Errorf("Adding subnet %s static route in network %q failed\n", s.subnetIP, n.id)
}
}
if iNames := jinfo.InterfaceName(); iNames != nil {
err = iNames.SetNames(containerIfName, "eth")
if err != nil {
return err
}
}
d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac,
net.ParseIP(d.advertiseAddress), true)
if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {
log.Warn(err)
}
buf, err := proto.Marshal(&PeerRecord{
EndpointIP: ep.addr.String(),
EndpointMAC: ep.mac.String(),
TunnelEndpointIP: d.advertiseAddress,
})
if err != nil {
return err
}
if err := jinfo.AddTableEntry(ovPeerTable, eid, buf); err != nil {
log.Errorf("overlay: Failed adding table entry to joininfo: %v", err)
}
d.pushLocalEndpointEvent("join", nid, eid)
return nil
}
// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
defer osl.InitOSContext()()
n, err := d.getNetwork(nid)
if err != nil {
return err
}
endpoint := n.endpoint(eid)
if endpoint == nil {
return fmt.Errorf("could not find endpoint with id %s", eid)
}
// generate a name for the iface that will be renamed to eth0 in the sbox
containerIfName, err := netutils.GenerateIfaceName(ns.NlHandle(), vethPrefix, vethLen)
if err != nil {
return fmt.Errorf("error generating an interface name: %s", err)
}
// create the netlink macvlan interface
vethName, err := createMacVlan(containerIfName, n.config.Parent, n.config.MacvlanMode)
if err != nil {
return err
}
// bind the generated iface name to the endpoint
endpoint.srcName = vethName
ep := n.endpoint(eid)
if ep == nil {
return fmt.Errorf("could not find endpoint with id %s", eid)
}
// parse and match the endpoint address with the available v4 subnets
if len(n.config.Ipv4Subnets) > 0 {
s := n.getSubnetforIPv4(ep.addr)
if s == nil {
return fmt.Errorf("could not find a valid ipv4 subnet for endpoint %s", eid)
}
v4gw, _, err := net.ParseCIDR(s.GwIP)
if err != nil {
return fmt.Errorf("gatway %s is not a valid ipv4 address: %v", s.GwIP, err)
}
err = jinfo.SetGateway(v4gw)
if err != nil {
return err
}
logrus.Debugf("Macvlan Endpoint Joined with IPv4_Addr: %s, Gateway: %s, MacVlan_Mode: %s, Parent: %s",
ep.addr.IP.String(), v4gw.String(), n.config.MacvlanMode, n.config.Parent)
}
// parse and match the endpoint address with the available v6 subnets
if len(n.config.Ipv6Subnets) > 0 {
s := n.getSubnetforIPv6(ep.addrv6)
if s == nil {
return fmt.Errorf("could not find a valid ipv6 subnet for endpoint %s", eid)
}
v6gw, _, err := net.ParseCIDR(s.GwIP)
if err != nil {
return fmt.Errorf("gatway %s is not a valid ipv6 address: %v", s.GwIP, err)
}
err = jinfo.SetGatewayIPv6(v6gw)
if err != nil {
return err
}
logrus.Debugf("Macvlan Endpoint Joined with IPv6_Addr: %s Gateway: %s MacVlan_Mode: %s, Parent: %s",
ep.addrv6.IP.String(), v6gw.String(), n.config.MacvlanMode, n.config.Parent)
}
iNames := jinfo.InterfaceName()
err = iNames.SetNames(vethName, containerVethPrefix)
if err != nil {
return err
}
return nil
}
// Spis and keys are sorted in such away the one in position 0 is the primary
func updateNodeKey(lIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, delIdx int) []*spi {
logrus.Debugf("Updating keys for node: %s (%d,%d,%d)", rIP, newIdx, priIdx, delIdx)
spis := idxs
logrus.Debugf("Current: %v", spis)
// add new
if newIdx != -1 {
spis = append(spis, &spi{
forward: buildSPI(lIP, rIP, curKeys[newIdx].tag),
reverse: buildSPI(rIP, lIP, curKeys[newIdx].tag),
})
}
if delIdx != -1 {
// -rSA0
programSA(lIP, rIP, spis[delIdx], nil, reverse, false)
}
if newIdx > -1 {
// +RSA2
programSA(lIP, rIP, spis[newIdx], curKeys[newIdx], reverse, true)
}
if priIdx > 0 {
// +fSA2
fSA2, _, _ := programSA(lIP, rIP, spis[priIdx], curKeys[priIdx], forward, true)
// +fSP2, -fSP1
fullMask := net.CIDRMask(8*len(fSA2.Src), 8*len(fSA2.Src))
fSP1 := &netlink.XfrmPolicy{
Src: &net.IPNet{IP: fSA2.Src, Mask: fullMask},
Dst: &net.IPNet{IP: fSA2.Dst, Mask: fullMask},
Dir: netlink.XFRM_DIR_OUT,
Proto: 17,
DstPort: 4789,
Mark: &netlink.XfrmMark{
Value: mark,
},
Tmpls: []netlink.XfrmPolicyTmpl{
{
Src: fSA2.Src,
Dst: fSA2.Dst,
Proto: netlink.XFRM_PROTO_ESP,
Mode: netlink.XFRM_MODE_TRANSPORT,
Spi: fSA2.Spi,
},
},
}
logrus.Debugf("Updating fSP{%s}", fSP1)
if err := ns.NlHandle().XfrmPolicyUpdate(fSP1); err != nil {
logrus.Warnf("Failed to update fSP{%s}: %v", fSP1, err)
}
// -fSA1
programSA(lIP, rIP, spis[0], nil, forward, false)
}
// swap
if priIdx > 0 {
swp := spis[0]
spis[0] = spis[priIdx]
spis[priIdx] = swp
}
// prune
if delIdx != -1 {
if delIdx == 0 {
delIdx = priIdx
}
spis = append(spis[:delIdx], spis[delIdx+1:]...)
}
logrus.Debugf("Updated: %v", spis)
return spis
}
func (d *driver) createNetwork(config *networkConfiguration) error {
var err error
defer osl.InitOSContext()()
networkList := d.getNetworks()
for i, nw := range networkList {
nw.Lock()
nwConfig := nw.config
nw.Unlock()
if err := nwConfig.Conflicts(config); err != nil {
if config.DefaultBridge {
// We encountered and identified a stale default network
// We must delete it as libnetwork is the source of thruth
// The default network being created must be the only one
// This can happen only from docker 1.12 on ward
logrus.Infof("Removing stale default bridge network %s (%s)", nwConfig.ID, nwConfig.BridgeName)
if err := d.DeleteNetwork(nwConfig.ID); err != nil {
logrus.Warnf("Failed to remove stale default network: %s (%s): %v. Will remove from store.", nwConfig.ID, nwConfig.BridgeName, err)
d.storeDelete(nwConfig)
}
networkList = append(networkList[:i], networkList[i+1:]...)
} else {
return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s",
config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error())
}
}
}
// Create and set network handler in driver
network := &bridgeNetwork{
id: config.ID,
endpoints: make(map[string]*bridgeEndpoint),
config: config,
portMapper: portmapper.New(),
driver: d,
}
d.Lock()
d.networks[config.ID] = network
d.Unlock()
// On failure make sure to reset driver network handler to nil
defer func() {
if err != nil {
d.Lock()
delete(d.networks, config.ID)
d.Unlock()
}
}()
// Initialize handle when needed
d.Lock()
if d.nlh == nil {
d.nlh = ns.NlHandle()
}
d.Unlock()
// Create or retrieve the bridge L3 interface
bridgeIface := newInterface(d.nlh, config)
network.bridge = bridgeIface
// Verify the network configuration does not conflict with previously installed
// networks. This step is needed now because driver might have now set the bridge
// name on this config struct. And because we need to check for possible address
// conflicts, so we need to check against operationa lnetworks.
if err = config.conflictsWithNetworks(config.ID, networkList); err != nil {
return err
}
setupNetworkIsolationRules := func(config *networkConfiguration, i *bridgeInterface) error {
if err := network.isolateNetwork(networkList, true); err != nil {
if err := network.isolateNetwork(networkList, false); err != nil {
logrus.Warnf("Failed on removing the inter-network iptables rules on cleanup: %v", err)
}
return err
}
network.registerIptCleanFunc(func() error {
nwList := d.getNetworks()
return network.isolateNetwork(nwList, false)
})
return nil
}
// Prepare the bridge setup configuration
bridgeSetup := newBridgeSetup(config, bridgeIface)
// If the bridge interface doesn't exist, we need to start the setup steps
// by creating a new device and assigning it an IPv4 address.
bridgeAlreadyExists := bridgeIface.exists()
if !bridgeAlreadyExists {
bridgeSetup.queueStep(setupDevice)
}
// Even if a bridge exists try to setup IPv4.
bridgeSetup.queueStep(setupBridgeIPv4)
enableIPv6Forwarding := d.config.EnableIPForwarding && config.AddressIPv6 != nil
// Conditionally queue setup steps depending on configuration values.
for _, step := range []struct {
Condition bool
Fn setupStep
}{
// Enable IPv6 on the bridge if required. We do this even for a
// previously existing bridge, as it may be here from a previous
// installation where IPv6 wasn't supported yet and needs to be
// assigned an IPv6 link-local address.
{config.EnableIPv6, setupBridgeIPv6},
// We ensure that the bridge has the expectedIPv4 and IPv6 addresses in
// the case of a previously existing device.
{bridgeAlreadyExists, setupVerifyAndReconcile},
// Enable IPv6 Forwarding
{enableIPv6Forwarding, setupIPv6Forwarding},
// Setup Loopback Adresses Routing
{!d.config.EnableUserlandProxy, setupLoopbackAdressesRouting},
// Setup IPTables.
{d.config.EnableIPTables, network.setupIPTables},
//We want to track firewalld configuration so that
//if it is started/reloaded, the rules can be applied correctly
{d.config.EnableIPTables, network.setupFirewalld},
// Setup DefaultGatewayIPv4
{config.DefaultGatewayIPv4 != nil, setupGatewayIPv4},
// Setup DefaultGatewayIPv6
{config.DefaultGatewayIPv6 != nil, setupGatewayIPv6},
// Add inter-network communication rules.
{d.config.EnableIPTables, setupNetworkIsolationRules},
//Configure bridge networking filtering if ICC is off and IP tables are enabled
{!config.EnableICC && d.config.EnableIPTables, setupBridgeNetFiltering},
} {
if step.Condition {
bridgeSetup.queueStep(step.Fn)
}
}
// Apply the prepared list of steps, and abort at the first error.
bridgeSetup.queueStep(setupDeviceUp)
if err = bridgeSetup.apply(); err != nil {
return err
}
return nil
}
