// Checks whether this network's configuration for the network with this id conflicts with any of the passed networks
func (c *networkConfiguration) conflictsWithNetworks(id types.UUID, others []*bridgeNetwork) error {
for _, nw := range others {
nw.Lock()
nwID := nw.id
nwConfig := nw.config
nwBridge := nw.bridge
nw.Unlock()
if nwID == id {
continue
}
// Verify the name (which may have been set by newInterface()) does not conflict with
// existing bridge interfaces. Ironically the system chosen name gets stored in the config...
// Basically we are checking if the two original configs were both empty.
if nwConfig.BridgeName == c.BridgeName {
return types.ForbiddenErrorf("conflicts with network %s (%s) by bridge name", nwID, nwConfig.BridgeName)
}
// If this network config specifies the AddressIPv4, we need
// to make sure it does not conflict with any previously allocated
// bridges. This could not be completely caught by the config conflict
// check, because networks which config does not specify the AddressIPv4
// get their address and subnet selected by the driver (see electBridgeIPv4())
if c.AddressIPv4 != nil {
if nwBridge.bridgeIPv4.Contains(c.AddressIPv4.IP) ||
c.AddressIPv4.Contains(nwBridge.bridgeIPv4.IP) {
return types.ForbiddenErrorf("conflicts with network %s (%s) by ip network", nwID, nwConfig.BridgeName)
}
}
}
return nil
}
func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {
ep.joinLeaveStart()
defer ep.joinLeaveEnd()
if sbox == nil || sbox.ID() == "" || sbox.Key() == "" {
return types.BadRequestErrorf("invalid Sandbox passed to enpoint leave: %v", sbox)
}
sb, ok := sbox.(*sandbox)
if !ok {
return types.BadRequestErrorf("not a valid Sandbox interface")
}
ep.Lock()
sid := ep.sandboxID
ep.Unlock()
if sid == "" {
return types.ForbiddenErrorf("cannot leave endpoint with no attached sandbox")
}
if sid != sbox.ID() {
return types.ForbiddenErrorf("unexpected sandbox ID in leave request. Expected %s. Got %s", ep.sandboxID, sbox.ID())
}
ep.processOptions(options...)
ep.Lock()
ep.sandboxID = ""
n := ep.network
ep.Unlock()
n.Lock()
c := n.ctrlr
d := n.driver
n.Unlock()
if err := c.updateEndpointToStore(ep); err != nil {
ep.Lock()
ep.sandboxID = sid
ep.Unlock()
return err
}
if err := d.Leave(n.id, ep.id); err != nil {
return err
}
if err := sb.clearNetworkResources(ep); err != nil {
return err
}
if sb.needDefaultGW() {
ep := sb.getEPwithoutGateway()
if ep == nil {
return fmt.Errorf("endpoint without GW expected, but not found")
}
return sb.setupDefaultGW(ep)
}
return sb.clearDefaultGW()
}
func (a *Allocator) initializeAddressSpace(as string, ds datastore.DataStore) error {
scope := ""
if ds != nil {
scope = ds.Scope()
}
a.Lock()
if currAS, ok := a.addrSpaces[as]; ok {
if currAS.ds != nil {
a.Unlock()
return types.ForbiddenErrorf("a datastore is already configured for the address space %s", as)
}
}
a.addrSpaces[as] = &addrSpace{
subnets: map[SubnetKey]*PoolData{},
id: dsConfigKey + "/" + as,
scope: scope,
ds: ds,
alloc: a,
}
a.Unlock()
a.checkConsistency(as)
return nil
}
func setAddress(ifaceAddr *string, address *net.IPNet) error {
if *ifaceAddr != "" {
return types.ForbiddenErrorf("endpoint interface IP present (%s). Cannot be modified with (%s).", *ifaceAddr, address)
}
*ifaceAddr = address.String()
return nil
}
func (r *DrvRegistry) registerIpamDriver(name string, driver ipamapi.Ipam, caps *ipamapi.Capability) error {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("ipam driver name string cannot be empty")
}
r.Lock()
_, ok := r.ipamDrivers[name]
r.Unlock()
if ok {
return types.ForbiddenErrorf("ipam driver %q already registered", name)
}
locAS, glbAS, err := driver.GetDefaultAddressSpaces()
if err != nil {
return types.InternalErrorf("ipam driver %q failed to return default address spaces: %v", name, err)
}
if r.ifn != nil {
if err := r.ifn(name, driver, caps); err != nil {
return err
}
}
r.Lock()
r.ipamDrivers[name] = &ipamData{driver: driver, defaultLocalAddressSpace: locAS, defaultGlobalAddressSpace: glbAS, capability: caps}
r.Unlock()
return nil
}
func (c *networkConfiguration) processIPAM(id string, ipamV4Data, ipamV6Data []driverapi.IPAMData) error {
if len(ipamV4Data) > 1 || len(ipamV6Data) > 1 {
return types.ForbiddenErrorf("bridge driver doesn't support multiple subnets")
}
if len(ipamV4Data) == 0 {
return types.BadRequestErrorf("bridge network %s requires ipv4 configuration", id)
}
if ipamV4Data[0].Gateway != nil {
c.AddressIPv4 = types.GetIPNetCopy(ipamV4Data[0].Gateway)
}
if gw, ok := ipamV4Data[0].AuxAddresses[DefaultGatewayV4AuxKey]; ok {
c.DefaultGatewayIPv4 = gw.IP
}
if len(ipamV6Data) > 0 {
c.AddressIPv6 = ipamV6Data[0].Pool
if ipamV6Data[0].Gateway != nil {
c.AddressIPv6 = types.GetIPNetCopy(ipamV6Data[0].Gateway)
}
if gw, ok := ipamV6Data[0].AuxAddresses[DefaultGatewayV6AuxKey]; ok {
c.DefaultGatewayIPv6 = gw.IP
}
}
return nil
}
// Create a new network using bridge plugin
func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error {
if len(ipV4Data) == 0 || ipV4Data[0].Pool.String() == "0.0.0.0/0" {
return types.BadRequestErrorf("ipv4 pool is empty")
}
// Sanity checks
d.Lock()
if _, ok := d.networks[id]; ok {
d.Unlock()
return types.ForbiddenErrorf("network %s exists", id)
}
d.Unlock()
// Parse and validate the config. It should not conflict with existing networks' config
config, err := parseNetworkOptions(id, option)
if err != nil {
return err
}
err = config.processIPAM(id, ipV4Data, ipV6Data)
if err != nil {
return err
}
if err = d.createNetwork(config); err != nil {
return err
}
return d.storeUpdate(config)
}
func setAddress(ifaceAddr **net.IPNet, address *net.IPNet) error {
if *ifaceAddr != nil {
return types.ForbiddenErrorf("endpoint interface IP present (%s). Cannot be modified with (%s).", *ifaceAddr, address)
}
*ifaceAddr = types.GetIPNetCopy(address)
return nil
}
func (sb *sandbox) SetKey(basePath string) error {
var err error
if basePath == "" {
return types.BadRequestErrorf("invalid sandbox key")
}
sb.Lock()
if sb.osSbox != nil {
sb.Unlock()
return types.ForbiddenErrorf("failed to set sandbox key : already assigned")
}
sb.Unlock()
osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key())
if err != nil {
return err
}
sb.Lock()
sb.osSbox = osSbox
sb.Unlock()
defer func() {
if err != nil {
sb.Lock()
sb.osSbox = nil
sb.Unlock()
}
}()
for _, ep := range sb.getConnectedEndpoints() {
if err = sb.populateNetworkResources(ep); err != nil {
return err
}
}
return nil
}
// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
switch dType {
case discoverapi.NodeDiscovery:
nodeData, ok := data.(discoverapi.NodeDiscoveryData)
if !ok || nodeData.Address == "" {
return fmt.Errorf("invalid discovery data")
}
d.nodeJoin(nodeData.Address, nodeData.Self)
case discoverapi.DatastoreConfig:
var err error
if d.store != nil {
return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already")
}
dsc, ok := data.(discoverapi.DatastoreConfigData)
if !ok {
return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
}
d.store, err = datastore.NewDataStoreFromConfig(dsc)
if err != nil {
return types.InternalErrorf("failed to initialize data store: %v", err)
}
default:
}
return nil
}
func (d *driver) setKeys(keys []*key) error {
if d.keys != nil {
return types.ForbiddenErrorf("initial keys are already present")
}
d.keys = keys
logrus.Debugf("Initial encryption keys: %v", d.keys)
return nil
}
// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
var err error
switch dType {
case discoverapi.NodeDiscovery:
nodeData, ok := data.(discoverapi.NodeDiscoveryData)
if !ok || nodeData.Address == "" {
return fmt.Errorf("invalid discovery data")
}
d.nodeJoin(nodeData.Address, nodeData.Self)
case discoverapi.DatastoreConfig:
if d.store != nil {
return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already")
}
dsc, ok := data.(discoverapi.DatastoreConfigData)
if !ok {
return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
}
d.store, err = datastore.NewDataStoreFromConfig(dsc)
if err != nil {
return types.InternalErrorf("failed to initialize data store: %v", err)
}
case discoverapi.EncryptionKeysConfig:
encrData, ok := data.(discoverapi.DriverEncryptionConfig)
if !ok {
return fmt.Errorf("invalid encryption key notification data")
}
keys := make([]*key, 0, len(encrData.Keys))
for i := 0; i < len(encrData.Keys); i++ {
k, err := parseEncryptionKey(encrData.Keys[i], encrData.Tags[i])
if err != nil {
return err
}
keys = append(keys, k)
}
d.setKeys(keys)
case discoverapi.EncryptionKeysUpdate:
var newKey, delKey, priKey *key
encrData, ok := data.(discoverapi.DriverEncryptionUpdate)
if !ok {
return fmt.Errorf("invalid encryption key notification data")
}
newKey, err = parseEncryptionKey(encrData.Key, encrData.Tag)
if err != nil {
return err
}
priKey, err = parseEncryptionKey(encrData.Primary, encrData.PrimaryTag)
if err != nil {
return err
}
delKey, err = parseEncryptionKey(encrData.Prune, encrData.PruneTag)
if err != nil {
return err
}
d.updateKeys(newKey, priKey, delKey)
default:
}
return nil
}
func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoint, error) {
var err error
if !config.IsValidName(name) {
return nil, ErrInvalidName(name)
}
if _, err = n.EndpointByName(name); err == nil {
return nil, types.ForbiddenErrorf("service endpoint with name %s already exists", name)
}
ep := &endpoint{name: name, generic: make(map[string]interface{}), iface: &endpointInterface{}}
ep.id = stringid.GenerateRandomID()
ep.network = n
ep.processOptions(options...)
if err = ep.assignAddress(); err != nil {
return nil, err
}
defer func() {
if err != nil {
ep.releaseAddress()
}
}()
ctrlr := n.getController()
n.IncEndpointCnt()
if err = ctrlr.updateToStore(n); err != nil {
return nil, err
}
defer func() {
if err != nil {
n.DecEndpointCnt()
if err = ctrlr.updateToStore(n); err != nil {
log.Warnf("endpoint count cleanup failed when updating network for %s : %v", name, err)
}
}
}()
if err = n.addEndpoint(ep); err != nil {
return nil, err
}
defer func() {
if err != nil {
if e := ep.Delete(); ep != nil {
log.Warnf("cleaning up endpoint failed %s : %v", name, e)
}
}
}()
if !ep.isLocalScoped() {
if err = ctrlr.updateToStore(ep); err != nil {
return nil, err
}
}
return ep, nil
}
func (i *testInterface) SetMacAddress(mac net.HardwareAddr) error {
if i.mac != nil {
return types.ForbiddenErrorf("endpoint interface MAC address present (%s). Cannot be modified with %s.", i.mac, mac)
}
if mac == nil {
return types.BadRequestErrorf("tried to set nil MAC address to endpoint interface")
}
i.mac = types.GetMacCopy(mac)
return nil
}
func (test *testEndpoint) SetMacAddress(mac net.HardwareAddr) error {
if test.macAddress != "" {
return types.ForbiddenErrorf("endpoint interface MAC address present (%s). Cannot be modified with %s.", test.macAddress, mac)
}
if mac == nil {
return types.BadRequestErrorf("tried to set nil MAC address to endpoint interface")
}
test.macAddress = mac.String()
return nil
}
func (sb *sandbox) Delete() error {
sb.Lock()
if sb.inDelete {
sb.Unlock()
return types.ForbiddenErrorf("another sandbox delete in progress")
}
// Set the inDelete flag. This will ensure that we don't
// update the store until we have completed all the endpoint
// leaves and deletes. And when endpoint leaves and deletes
// are completed then we can finally delete the sandbox object
// altogether from the data store. If the daemon exits
// ungracefully in the middle of a sandbox delete this way we
// will have all the references to the endpoints in the
// sandbox so that we can clean them up when we restart
sb.inDelete = true
sb.Unlock()
c := sb.controller
// Detach from all endpoints
for _, ep := range sb.getConnectedEndpoints() {
// endpoint in the Gateway network will be cleaned up
// when when sandbox no longer needs external connectivity
if ep.endpointInGWNetwork() {
continue
}
if err := ep.Leave(sb); err != nil {
log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
}
if err := ep.Delete(); err != nil {
log.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err)
}
}
// Container is going away. Path cache in etchosts is most
// likely not required any more. Drop it.
etchosts.Drop(sb.config.hostsPath)
if sb.osSbox != nil {
sb.osSbox.Destroy()
}
if err := sb.storeDelete(); err != nil {
log.Warnf("Failed to delete sandbox %s from store: %v", sb.ID(), err)
}
c.Lock()
delete(c.sandboxes, sb.ID())
c.Unlock()
return nil
}
func (c *networkConfiguration) processIPAM(id string, ipamV4Data, ipamV6Data []driverapi.IPAMData) error {
if len(ipamV6Data) > 0 {
return types.ForbiddenErrorf("windowsshim driver doesnt support v6 subnets")
}
if len(ipamV4Data) == 0 {
return types.BadRequestErrorf("network %s requires ipv4 configuration", id)
}
return nil
}
func (epi *endpointInterface) SetIPAliases(aliases []*net.IPNet) error {
if epi.ipAliases != nil {
return types.ForbiddenErrorf("endpoint aliases present. Cannot be modified with (%s).", aliases)
}
epi.ipAliases = make([]*net.IPNet, 0)
for _, alias := range aliases {
epi.ipAliases = append(epi.ipAliases, types.GetIPNetCopy(alias))
}
return nil
}
func (d *driver) CreateNetwork(id string, option map[string]interface{}) error {
d.Lock()
defer d.Unlock()
if d.network != "" {
return types.ForbiddenErrorf("only one instance of \"%s\" network is allowed", networkType)
}
d.network = id
return nil
}
func (ep *endpoint) SetIPAddress(address *net.IPNet) error {
if address.IP == nil {
return types.BadRequestErrorf("tried to set nil IP address to endpoint interface")
}
if address.IP.To4() == nil {
return types.NotImplementedErrorf("do not support ipv6 yet")
}
if ep.addr != nil {
return types.ForbiddenErrorf("endpoint interface IP present (%s). Cannot be modified with %s.", ep.addr, address)
}
ep.addr = types.GetIPNetCopy(address)
return nil
}
func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal, add bool) error {
logrus.Debugf("checkEncryption(%s, %v, %d, %t)", nid[0:7], rIP, vxlanID, isLocal)
n := d.network(nid)
if n == nil || !n.secure {
return nil
}
if len(d.keys) == 0 {
return types.ForbiddenErrorf("encryption key is not present")
}
lIP := net.ParseIP(d.bindAddress)
aIP := net.ParseIP(d.advertiseAddress)
nodes := map[string]net.IP{}
switch {
case isLocal:
if err := d.peerDbNetworkWalk(nid, func(pKey *peerKey, pEntry *peerEntry) bool {
if !aIP.Equal(pEntry.vtep) {
nodes[pEntry.vtep.String()] = pEntry.vtep
}
return false
}); err != nil {
logrus.Warnf("Failed to retrieve list of participating nodes in overlay network %s: %v", nid[0:5], err)
}
default:
if len(d.network(nid).endpoints) > 0 {
nodes[rIP.String()] = rIP
}
}
logrus.Debugf("List of nodes: %s", nodes)
if add {
for _, rIP := range nodes {
if err := setupEncryption(lIP, aIP, rIP, vxlanID, d.secMap, d.keys); err != nil {
logrus.Warnf("Failed to program network encryption between %s and %s: %v", lIP, rIP, err)
}
}
} else {
if len(nodes) == 0 {
if err := removeEncryption(lIP, rIP, d.secMap); err != nil {
logrus.Warnf("Failed to remove network encryption between %s and %s: %v", lIP, rIP, err)
}
}
}
return nil
}
func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoint, error) {
var err error
if !config.IsValidName(name) {
return nil, ErrInvalidName(name)
}
if _, err = n.EndpointByName(name); err == nil {
return nil, types.ForbiddenErrorf("service endpoint with name %s already exists", name)
}
ep := &endpoint{name: name,
iFaces: []*endpointInterface{},
generic: make(map[string]interface{})}
ep.id = types.UUID(stringid.GenerateRandomID())
ep.network = n
ep.processOptions(options...)
n.Lock()
ctrlr := n.ctrlr
n.Unlock()
n.IncEndpointCnt()
if err = ctrlr.updateNetworkToStore(n); err != nil {
return nil, err
}
defer func() {
if err != nil {
n.DecEndpointCnt()
if err = ctrlr.updateNetworkToStore(n); err != nil {
log.Warnf("endpoint count cleanup failed when updating network for %s : %v", name, err)
}
}
}()
if err = n.addEndpoint(ep); err != nil {
return nil, err
}
defer func() {
if err != nil {
if e := ep.Delete(); ep != nil {
log.Warnf("cleaning up endpoint failed %s : %v", name, e)
}
}
}()
if err = ctrlr.updateEndpointToStore(ep); err != nil {
return nil, err
}
return ep, nil
}
func (a *Allocator) initializeAddressSpace(as string, ds datastore.DataStore) error {
a.Lock()
if _, ok := a.addrSpaces[as]; ok {
a.Unlock()
return types.ForbiddenErrorf("tried to add an axisting address space: %s", as)
}
a.addrSpaces[as] = &addrSpace{
subnets: map[SubnetKey]*PoolData{},
id: dsConfigKey + "/" + as,
scope: ds.Scope(),
ds: ds,
alloc: a,
}
a.Unlock()
a.checkConsistency(as)
return nil
}
func (c *networkConfig) parseIPAM(id string, ipamV4Data, ipamV6Data []*driverapi.IPAMData) error {
if len(ipamV4Data) > 1 || len(ipamV6Data) > 1 {
return types.ForbiddenErrorf("bridge driver doesni't support multiple subnets")
}
if len(ipamV4Data) == 0 {
return types.BadRequestErrorf("bridge network %s requires ipv4 configuration", id)
}
if ipamV4Data[0].Gateway == "" {
return types.BadRequestErrorf("bridge network %s requires ipv4 gateway from IPAM", id)
}
if gw, _, err := net.ParseCIDR(ipamV4Data[0].Gateway); err != nil {
return err
} else {
Log.Debugf("IPAM: %v", *ipamV4Data[0])
c.GatewayIPv4 = gw
}
return nil
}
func (c *controller) registerIpamDriver(name string, driver ipamapi.Ipam, caps *ipamapi.Capability) error {
if !config.IsValidName(name) {
return ErrInvalidName(name)
}
c.Lock()
_, ok := c.ipamDrivers[name]
c.Unlock()
if ok {
return types.ForbiddenErrorf("ipam driver %q already registered", name)
}
locAS, glbAS, err := driver.GetDefaultAddressSpaces()
if err != nil {
return types.InternalErrorf("ipam driver %q failed to return default address spaces: %v", name, err)
}
c.Lock()
c.ipamDrivers[name] = &ipamData{driver: driver, defaultLocalAddressSpace: locAS, defaultGlobalAddressSpace: glbAS, capability: caps}
c.Unlock()
log.Debugf("Registering ipam driver: %q", name)
return nil
}
func (d *driver) DeleteNetwork(nid types.UUID) error {
var err error
defer sandbox.InitOSContext()()
// Get network handler and remove it from driver
d.Lock()
n, ok := d.networks[nid]
d.Unlock()
if !ok {
return types.InternalMaskableErrorf("network %s does not exist", nid)
}
n.Lock()
config := n.config
n.Unlock()
if config.BridgeName == DefaultBridgeName {
return types.ForbiddenErrorf("default network of type \"%s\" cannot be deleted", networkType)
}
d.Lock()
delete(d.networks, nid)
d.Unlock()
// On failure set network handler back in driver, but
// only if is not already taken over by some other thread
defer func() {
if err != nil {
d.Lock()
if _, ok := d.networks[nid]; !ok {
d.networks[nid] = n
}
d.Unlock()
}
}()
// Sanity check
if n == nil {
err = driverapi.ErrNoNetwork(nid)
return err
}
// Cannot remove network if endpoints are still present
if len(n.endpoints) != 0 {
err = ActiveEndpointsError(n.id)
return err
}
// In case of failures after this point, restore the network isolation rules
nwList := d.getNetworks()
defer func() {
if err != nil {
if err := n.isolateNetwork(nwList, true); err != nil {
logrus.Warnf("Failed on restoring the inter-network iptables rules on cleanup: %v", err)
}
}
}()
// Remove inter-network communication rules.
err = n.isolateNetwork(nwList, false)
if err != nil {
return err
}
// Programming
err = netlink.LinkDel(n.bridge.Link)
return err
}
// Create a new network using bridge plugin
func (d *driver) CreateNetwork(id types.UUID, option map[string]interface{}) error {
var err error
defer sandbox.InitOSContext()()
// Sanity checks
d.Lock()
if _, ok := d.networks[id]; ok {
d.Unlock()
return types.ForbiddenErrorf("network %s exists", id)
}
d.Unlock()
// Parse and validate the config. It should not conflict with existing networks' config
config, err := parseNetworkOptions(option)
if err != nil {
return err
}
networkList := d.getNetworks()
for _, nw := range networkList {
nw.Lock()
nwConfig := nw.config
nw.Unlock()
if nwConfig.Conflicts(config) {
return types.ForbiddenErrorf("conflicts with network %s (%s)", nw.id, nw.config.BridgeName)
}
}
// Create and set network handler in driver
network := &bridgeNetwork{
id: id,
endpoints: make(map[types.UUID]*bridgeEndpoint),
config: config,
portMapper: portmapper.New(),
}
d.Lock()
d.networks[id] = network
d.Unlock()
// On failure make sure to reset driver network handler to nil
defer func() {
if err != nil {
d.Lock()
delete(d.networks, id)
d.Unlock()
}
}()
// Create or retrieve the bridge L3 interface
bridgeIface := newInterface(config)
network.bridge = bridgeIface
// Verify the network configuration does not conflict with previously installed
// networks. This step is needed now because driver might have now set the bridge
// name on this config struct. And because we need to check for possible address
// conflicts, so we need to check against operationa lnetworks.
if err = config.conflictsWithNetworks(id, networkList); err != nil {
return err
}
setupNetworkIsolationRules := func(config *networkConfiguration, i *bridgeInterface) error {
if err := network.isolateNetwork(networkList, true); err != nil {
if err := network.isolateNetwork(networkList, false); err != nil {
logrus.Warnf("Failed on removing the inter-network iptables rules on cleanup: %v", err)
}
return err
}
return nil
}
// Prepare the bridge setup configuration
bridgeSetup := newBridgeSetup(config, bridgeIface)
// If the bridge interface doesn't exist, we need to start the setup steps
// by creating a new device and assigning it an IPv4 address.
bridgeAlreadyExists := bridgeIface.exists()
if !bridgeAlreadyExists {
bridgeSetup.queueStep(setupDevice)
}
// Even if a bridge exists try to setup IPv4.
bridgeSetup.queueStep(setupBridgeIPv4)
enableIPv6Forwarding := false
if d.config != nil && d.config.EnableIPForwarding && config.FixedCIDRv6 != nil {
enableIPv6Forwarding = true
}
// Conditionally queue setup steps depending on configuration values.
for _, step := range []struct {
Condition bool
Fn setupStep
}{
// Enable IPv6 on the bridge if required. We do this even for a
// previously existing bridge, as it may be here from a previous
// installation where IPv6 wasn't supported yet and needs to be
// assigned an IPv6 link-local address.
{config.EnableIPv6, setupBridgeIPv6},
// We ensure that the bridge has the expectedIPv4 and IPv6 addresses in
// the case of a previously existing device.
{bridgeAlreadyExists, setupVerifyAndReconcile},
// Setup the bridge to allocate containers IPv4 addresses in the
// specified subnet.
{config.FixedCIDR != nil, setupFixedCIDRv4},
// Setup the bridge to allocate containers global IPv6 addresses in the
// specified subnet.
{config.FixedCIDRv6 != nil, setupFixedCIDRv6},
// Enable IPv6 Forwarding
{enableIPv6Forwarding, setupIPv6Forwarding},
// Setup Loopback Adresses Routing
{!config.EnableUserlandProxy, setupLoopbackAdressesRouting},
// Setup IPTables.
{config.EnableIPTables, network.setupIPTables},
//We want to track firewalld configuration so that
//if it is started/reloaded, the rules can be applied correctly
{config.EnableIPTables, network.setupFirewalld},
// Setup DefaultGatewayIPv4
{config.DefaultGatewayIPv4 != nil, setupGatewayIPv4},
// Setup DefaultGatewayIPv6
{config.DefaultGatewayIPv6 != nil, setupGatewayIPv6},
// Add inter-network communication rules.
{config.EnableIPTables, setupNetworkIsolationRules},
//Configure bridge networking filtering if ICC is off and IP tables are enabled
{!config.EnableICC && config.EnableIPTables, setupBridgeNetFiltering},
} {
if step.Condition {
bridgeSetup.queueStep(step.Fn)
}
}
// Block bridge IP from being allocated.
bridgeSetup.queueStep(allocateBridgeIP)
// Apply the prepared list of steps, and abort at the first error.
bridgeSetup.queueStep(setupDeviceUp)
if err = bridgeSetup.apply(); err != nil {
return err
}
return nil
}
// Create a new network
func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error {
if _, err := d.getNetwork(id); err == nil {
return types.ForbiddenErrorf("network %s exists", id)
}
genData, ok := option[netlabel.GenericData].(map[string]string)
if !ok {
return fmt.Errorf("Unknown generic data option")
}
// Parse and validate the config. It should not conflict with existing networks' config
config, err := d.parseNetworkOptions(id, genData)
if err != nil {
return err
}
err = config.processIPAM(id, ipV4Data, ipV6Data)
if err != nil {
return err
}
network := &hnsNetwork{
id: config.ID,
endpoints: make(map[string]*hnsEndpoint),
config: config,
driver: d,
}
d.Lock()
d.networks[config.ID] = network
d.Unlock()
// A non blank hnsid indicates that the network was discovered
// from HNS. No need to call HNS if this network was discovered
// from HNS
if config.HnsID == "" {
subnets := []hcsshim.Subnet{}
for _, ipData := range ipV4Data {
subnet := hcsshim.Subnet{
AddressPrefix: ipData.Pool.String(),
GatewayAddress: ipData.Gateway.IP.String(),
}
subnets = append(subnets, subnet)
}
network := &hcsshim.HNSNetwork{
Name: config.Name,
Type: d.name,
Subnets: subnets,
}
if network.Name == "" {
network.Name = id
}
configurationb, err := json.Marshal(network)
if err != nil {
return err
}
configuration := string(configurationb)
log.Debugf("HNSNetwork Request =%v Address Space=%v", configuration, subnets)
hnsresponse, err := hcsshim.HNSNetworkRequest("POST", "", configuration)
if err != nil {
return err
}
config.HnsID = hnsresponse.Id
genData[HNSID] = config.HnsID
}
return nil
}
func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error {
n, err := ep.getNetworkFromStore()
if err != nil {
return fmt.Errorf("failed to get network from store during join: %v", err)
}
ep, err = n.getEndpointFromStore(ep.ID())
if err != nil {
return fmt.Errorf("failed to get endpoint from store during join: %v", err)
}
ep.Lock()
if ep.sandboxID != "" {
ep.Unlock()
return types.ForbiddenErrorf("another container is attached to the same network endpoint")
}
ep.network = n
ep.sandboxID = sb.ID()
ep.joinInfo = &endpointJoinInfo{}
epid := ep.id
ep.Unlock()
defer func() {
if err != nil {
ep.Lock()
ep.sandboxID = ""
ep.Unlock()
}
}()
nid := n.ID()
ep.processOptions(options...)
d, err := n.driver(true)
if err != nil {
return fmt.Errorf("failed to join endpoint: %v", err)
}
err = d.Join(nid, epid, sb.Key(), ep, sb.Labels())
if err != nil {
return err
}
defer func() {
if err != nil {
if err := d.Leave(nid, epid); err != nil {
log.Warnf("driver leave failed while rolling back join: %v", err)
}
}
}()
// Watch for service records
if !n.getController().isAgent() {
n.getController().watchSvcRecord(ep)
}
if doUpdateHostsFile(n, sb) {
address := ""
if ip := ep.getFirstInterfaceAddress(); ip != nil {
address = ip.String()
}
if err = sb.updateHostsFile(address); err != nil {
return err
}
}
if err = sb.updateDNS(n.enableIPv6); err != nil {
return err
}
if err = n.getController().updateToStore(ep); err != nil {
return err
}
// Current endpoint providing external connectivity for the sandbox
extEp := sb.getGatewayEndpoint()
sb.Lock()
heap.Push(&sb.endpoints, ep)
sb.Unlock()
defer func() {
if err != nil {
sb.removeEndpoint(ep)
}
}()
if err = sb.populateNetworkResources(ep); err != nil {
return err
}
if e := ep.addToCluster(); e != nil {
log.Errorf("Could not update state for endpoint %s into cluster: %v", ep.Name(), e)
}
if sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
return sb.setupDefaultGW()
}
moveExtConn := sb.getGatewayEndpoint() != extEp
if moveExtConn {
if extEp != nil {
log.Debugf("Revoking external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
extN, err := extEp.getNetworkFromStore()
if err != nil {
return fmt.Errorf("failed to get network from store during join: %v", err)
}
extD, err := extN.driver(true)
if err != nil {
return fmt.Errorf("failed to join endpoint: %v", err)
}
if err = extD.RevokeExternalConnectivity(extEp.network.ID(), extEp.ID()); err != nil {
return types.InternalErrorf(
"driver failed revoking external connectivity on endpoint %s (%s): %v",
extEp.Name(), extEp.ID(), err)
}
defer func() {
if err != nil {
if e := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); e != nil {
log.Warnf("Failed to roll-back external connectivity on endpoint %s (%s): %v",
extEp.Name(), extEp.ID(), e)
}
}
}()
}
if !n.internal {
log.Debugf("Programming external connectivity on endpoint %s (%s)", ep.Name(), ep.ID())
if err = d.ProgramExternalConnectivity(n.ID(), ep.ID(), sb.Labels()); err != nil {
return types.InternalErrorf(
"driver failed programming external connectivity on endpoint %s (%s): %v",
ep.Name(), ep.ID(), err)
}
}
}
if !sb.needDefaultGW() {
if err := sb.clearDefaultGW(); err != nil {
log.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v",
sb.ID(), sb.ContainerID(), err)
}
}
return nil
}
func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption) error {
n, err := ep.getNetworkFromStore()
if err != nil {
return fmt.Errorf("failed to get network from store during leave: %v", err)
}
ep, err = n.getEndpointFromStore(ep.ID())
if err != nil {
return fmt.Errorf("failed to get endpoint from store during leave: %v", err)
}
ep.Lock()
sid := ep.sandboxID
ep.Unlock()
if sid == "" {
return types.ForbiddenErrorf("cannot leave endpoint with no attached sandbox")
}
if sid != sb.ID() {
return types.ForbiddenErrorf("unexpected sandbox ID in leave request. Expected %s. Got %s", ep.sandboxID, sb.ID())
}
ep.processOptions(options...)
d, err := n.driver(!force)
if err != nil {
return fmt.Errorf("failed to leave endpoint: %v", err)
}
ep.Lock()
ep.sandboxID = ""
ep.network = n
ep.Unlock()
// Current endpoint providing external connectivity to the sandbox
extEp := sb.getGatewayEndpoint()
moveExtConn := extEp != nil && (extEp.ID() == ep.ID())
if d != nil {
if moveExtConn {
log.Debugf("Revoking external connectivity on endpoint %s (%s)", ep.Name(), ep.ID())
if err := d.RevokeExternalConnectivity(n.id, ep.id); err != nil {
log.Warnf("driver failed revoking external connectivity on endpoint %s (%s): %v",
ep.Name(), ep.ID(), err)
}
}
if err := d.Leave(n.id, ep.id); err != nil {
if _, ok := err.(types.MaskableError); !ok {
log.Warnf("driver error disconnecting container %s : %v", ep.name, err)
}
}
}
if err := sb.clearNetworkResources(ep); err != nil {
log.Warnf("Could not cleanup network resources on container %s disconnect: %v", ep.name, err)
}
// Update the store about the sandbox detach only after we
// have completed sb.clearNetworkresources above to avoid
// spurious logs when cleaning up the sandbox when the daemon
// ungracefully exits and restarts before completing sandbox
// detach but after store has been updated.
if err := n.getController().updateToStore(ep); err != nil {
return err
}
if e := ep.deleteFromCluster(); e != nil {
log.Errorf("Could not delete state for endpoint %s from cluster: %v", ep.Name(), e)
}
sb.deleteHostsEntries(n.getSvcRecords(ep))
if !sb.inDelete && sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
return sb.setupDefaultGW()
}
// New endpoint providing external connectivity for the sandbox
extEp = sb.getGatewayEndpoint()
if moveExtConn && extEp != nil {
log.Debugf("Programming external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
extN, err := extEp.getNetworkFromStore()
if err != nil {
return fmt.Errorf("failed to get network from store during leave: %v", err)
}
extD, err := extN.driver(true)
if err != nil {
return fmt.Errorf("failed to leave endpoint: %v", err)
}
if err := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); err != nil {
log.Warnf("driver failed programming external connectivity on endpoint %s: (%s) %v",
extEp.Name(), extEp.ID(), err)
}
}
if !sb.needDefaultGW() {
if err := sb.clearDefaultGW(); err != nil {
log.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v",
sb.ID(), sb.ContainerID(), err)
}
}
return nil
}
