func (n *network) initSandbox() error { n.Lock() n.initEpoch++ n.Unlock() sbox, err := osl.NewSandbox( osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), true) if err != nil { return fmt.Errorf("could not create network sandbox: %v", err) } n.setSandbox(sbox) n.driver.peerDbUpdateSandbox(n.id) var nlSock *nl.NetlinkSocket sbox.InvokeFunc(func() { nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) if err != nil { err = fmt.Errorf("failed to subscribe to neighbor group netlink messages") } }) go n.watchMiss(nlSock) return nil }
func (n *network) initSandbox() error { n.Lock() n.initEpoch++ n.Unlock() sbox, err := osl.NewSandbox( osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), true) if err != nil { return fmt.Errorf("could not create network sandbox: %v", err) } // Add a bridge inside the namespace if err := sbox.AddInterface("bridge1", "br", sbox.InterfaceOptions().Address(bridgeIP), sbox.InterfaceOptions().Bridge(true)); err != nil { return fmt.Errorf("could not create bridge inside the network sandbox: %v", err) } n.setSandbox(sbox) var nlSock *nl.NetlinkSocket sbox.InvokeFunc(func() { nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) if err != nil { err = fmt.Errorf("failed to subscribe to neighbor group netlink messages") } }) go n.watchMiss(nlSock) return n.initVxlan() }
func (n *network) initSandbox(restore bool) error { n.Lock() n.initEpoch++ n.Unlock() networkOnce.Do(networkOnceInit) if !restore { if hostMode { if err := addNetworkChain(n.id[:12]); err != nil { return err } } // If there are any stale sandboxes related to this network // from previous daemon life clean it up here n.cleanupStaleSandboxes() } // In the restore case network sandbox already exist; but we don't know // what epoch number it was created with. It has to be retrieved by // searching the net namespaces. key := "" if restore { key = osl.GenerateKey("-" + n.id) } else { key = osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch) + n.id) } sbox, err := osl.NewSandbox(key, !hostMode, restore) if err != nil { return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err) } n.setSandbox(sbox) if !restore { n.driver.peerDbUpdateSandbox(n.id) } var nlSock *nl.NetlinkSocket sbox.InvokeFunc(func() { nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) if err != nil { err = fmt.Errorf("failed to subscribe to neighbor group netlink messages") } }) if nlSock != nil { go n.watchMiss(nlSock) } return nil }
// NewSandbox creates a new sandbox for the passed container id func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) { var err error if containerID == "" { return nil, types.BadRequestErrorf("invalid container ID") } var existing Sandbox look := SandboxContainerWalker(&existing, containerID) c.WalkSandboxes(look) if existing != nil { return nil, types.BadRequestErrorf("container %s is already present: %v", containerID, existing) } // Create sandbox and process options first. Key generation depends on an option sb := &sandbox{ id: stringid.GenerateRandomID(), containerID: containerID, endpoints: epHeap{}, epPriority: map[string]int{}, config: containerConfig{}, controller: c, } // This sandbox may be using an existing osl sandbox, sharing it with another sandbox var peerSb Sandbox c.WalkSandboxes(SandboxKeyWalker(&peerSb, sb.Key())) if peerSb != nil { sb.osSbox = peerSb.(*sandbox).osSbox } heap.Init(&sb.endpoints) sb.processOptions(options...) if err = sb.setupResolutionFiles(); err != nil { return nil, err } if sb.osSbox == nil && !sb.config.useExternalKey { if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil { return nil, fmt.Errorf("failed to create new osl sandbox: %v", err) } } c.Lock() c.sandboxes[sb.id] = sb c.Unlock() return sb, nil }
func (n *network) initSandbox() error { n.Lock() n.initEpoch++ n.Unlock() networkOnce.Do(networkOnceInit) if hostMode { if err := addNetworkChain(n.id[:12]); err != nil { return err } } // If there are any stale sandboxes related to this network // from previous daemon life clean it up here n.cleanupStaleSandboxes() sbox, err := osl.NewSandbox( osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode) if err != nil { return fmt.Errorf("could not create network sandbox: %v", err) } n.setSandbox(sbox) n.driver.peerDbUpdateSandbox(n.id) var nlSock *nl.NetlinkSocket sbox.InvokeFunc(func() { nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) if err != nil { err = fmt.Errorf("failed to subscribe to neighbor group netlink messages") } }) if nlSock != nil { go n.watchMiss(nlSock) } return nil }
func (n *network) initSandbox(restore bool) error { n.Lock() n.initEpoch++ n.Unlock() networkOnce.Do(networkOnceInit) if !restore { // If there are any stale sandboxes related to this network // from previous daemon life clean it up here n.cleanupStaleSandboxes() } // In the restore case network sandbox already exist; but we don't know // what epoch number it was created with. It has to be retrieved by // searching the net namespaces. key := "" if restore { key = osl.GenerateKey("-" + n.id) } else { key = osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch) + n.id) } sbox, err := osl.NewSandbox(key, !hostMode, restore) if err != nil { return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err) } n.setSandbox(sbox) if !restore { n.driver.peerDbUpdateSandbox(n.id) } return nil }
// NewSandbox creates a new sandbox for the passed container id func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (sBox Sandbox, err error) { if containerID == "" { return nil, types.BadRequestErrorf("invalid container ID") } var sb *sandbox c.Lock() for _, s := range c.sandboxes { if s.containerID == containerID { // If not a stub, then we already have a complete sandbox. if !s.isStub { c.Unlock() return nil, types.ForbiddenErrorf("container %s is already present: %v", containerID, s) } // We already have a stub sandbox from the // store. Make use of it so that we don't lose // the endpoints from store but reset the // isStub flag. sb = s sb.isStub = false break } } c.Unlock() // Create sandbox and process options first. Key generation depends on an option if sb == nil { sb = &sandbox{ id: stringid.GenerateRandomID(), containerID: containerID, endpoints: epHeap{}, epPriority: map[string]int{}, populatedEndpoints: map[string]struct{}{}, config: containerConfig{}, controller: c, } } sBox = sb heap.Init(&sb.endpoints) sb.processOptions(options...) c.Lock() if sb.ingress && c.ingressSandbox != nil { c.Unlock() return nil, types.ForbiddenErrorf("ingress sandbox already present") } if sb.ingress { c.ingressSandbox = sb } c.Unlock() defer func() { if err != nil { c.Lock() if sb.ingress { c.ingressSandbox = nil } c.Unlock() } }() if err = sb.setupResolutionFiles(); err != nil { return nil, err } if sb.config.useDefaultSandBox { c.sboxOnce.Do(func() { c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false) }) if err != nil { c.sboxOnce = sync.Once{} return nil, fmt.Errorf("failed to create default sandbox: %v", err) } sb.osSbox = c.defOsSbox } if sb.osSbox == nil && !sb.config.useExternalKey { if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil { return nil, fmt.Errorf("failed to create new osl sandbox: %v", err) } } c.Lock() c.sandboxes[sb.id] = sb c.Unlock() defer func() { if err != nil { c.Lock() delete(c.sandboxes, sb.id) c.Unlock() } }() err = sb.storeUpdate() if err != nil { return nil, fmt.Errorf("updating the store state of sandbox failed: %v", err) } return sb, nil }
func (c *controller) sandboxCleanup() { store := c.getStore(datastore.LocalScope) if store == nil { logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes") return } kvol, err := store.List(datastore.Key(sandboxPrefix), &sbState{c: c}) if err != nil && err != datastore.ErrKeyNotFound { logrus.Errorf("failed to get sandboxes for scope %s: %v", store.Scope(), err) return } // It's normal for no sandboxes to be found. Just bail out. if err == datastore.ErrKeyNotFound { return } for _, kvo := range kvol { sbs := kvo.(*sbState) sb := &sandbox{ id: sbs.ID, controller: sbs.c, containerID: sbs.Cid, endpoints: epHeap{}, epPriority: map[string]int{}, dbIndex: sbs.dbIndex, isStub: true, dbExists: true, } sb.osSbox, err = osl.NewSandbox(sb.Key(), true) if err != nil { logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err) continue } c.Lock() c.sandboxes[sb.id] = sb c.Unlock() for _, eps := range sbs.Eps { n, err := c.getNetworkFromStore(eps.Nid) var ep *endpoint if err != nil { logrus.Errorf("getNetworkFromStore for nid %s failed while trying to build sandbox for cleanup: %v", eps.Nid, err) n = &network{id: eps.Nid, ctrlr: c, drvOnce: &sync.Once{}, persist: true} ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID} } else { ep, err = n.getEndpointFromStore(eps.Eid) if err != nil { logrus.Errorf("getEndpointFromStore for eid %s failed while trying to build sandbox for cleanup: %v", eps.Eid, err) ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID} } } heap.Push(&sb.endpoints, ep) } logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID) if err := sb.delete(true); err != nil { logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) } } }
func externalKeyTest(t *testing.T, reexec bool) { if !testutils.IsRunningInContainer() { defer testutils.SetupTestOSContext(t)() } n, err := createTestNetwork(bridgeNetType, "testnetwork", options.Generic{ netlabel.GenericData: options.Generic{ "BridgeName": "testnetwork", }, }, nil, nil) if err != nil { t.Fatal(err) } defer func() { if err := n.Delete(); err != nil { t.Fatal(err) } }() ep, err := n.CreateEndpoint("ep1") if err != nil { t.Fatal(err) } defer func() { err = ep.Delete() if err != nil { t.Fatal(err) } }() ep2, err := n.CreateEndpoint("ep2") if err != nil { t.Fatal(err) } defer func() { err = ep2.Delete() if err != nil { t.Fatal(err) } }() cnt, err := controller.NewSandbox(containerID, libnetwork.OptionHostname("test"), libnetwork.OptionDomainname("docker.io"), libnetwork.OptionUseExternalKey(), libnetwork.OptionExtraHost("web", "192.168.0.1")) defer func() { if err := cnt.Delete(); err != nil { t.Fatal(err) } osl.GC() }() // Join endpoint to sandbox before SetKey err = ep.Join(cnt) runtime.LockOSThread() if err != nil { t.Fatal(err) } defer func() { err = ep.Leave(cnt) runtime.LockOSThread() if err != nil { t.Fatal(err) } }() sbox := ep.Info().Sandbox() if sbox == nil { t.Fatalf("Expected to have a valid Sandbox") } if reexec { err := reexecSetKey("this-must-fail", containerID, controller.ID()) if err == nil { t.Fatalf("SetExternalKey must fail if the corresponding namespace is not created") } } else { // Setting an non-existing key (namespace) must fail if err := sbox.SetKey("this-must-fail"); err == nil { t.Fatalf("Setkey must fail if the corresponding namespace is not created") } } // Create a new OS sandbox using the osl API before using it in SetKey if extOsBox, err := osl.NewSandbox("ValidKey", true); err != nil { t.Fatalf("Failed to create new osl sandbox") } else { defer func() { if err := extOsBox.Destroy(); err != nil { log.Warnf("Failed to remove os sandbox: %v", err) } }() } if reexec { err := reexecSetKey("ValidKey", containerID, controller.ID()) if err != nil { t.Fatalf("SetExternalKey failed with %v", err) } } else { if err := sbox.SetKey("ValidKey"); err != nil { t.Fatalf("Setkey failed with %v", err) } } // Join endpoint to sandbox after SetKey err = ep2.Join(sbox) if err != nil { t.Fatal(err) } runtime.LockOSThread() defer func() { err = ep2.Leave(sbox) runtime.LockOSThread() if err != nil { t.Fatal(err) } }() if ep.Info().Sandbox().Key() != ep2.Info().Sandbox().Key() { t.Fatalf("ep1 and ep2 returned different container sandbox key") } checkSandbox(t, ep.Info()) }
func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) { store := c.getStore(datastore.LocalScope) if store == nil { logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes") return } kvol, err := store.List(datastore.Key(sandboxPrefix), &sbState{c: c}) if err != nil && err != datastore.ErrKeyNotFound { logrus.Errorf("failed to get sandboxes for scope %s: %v", store.Scope(), err) return } // It's normal for no sandboxes to be found. Just bail out. if err == datastore.ErrKeyNotFound { return } for _, kvo := range kvol { sbs := kvo.(*sbState) sb := &sandbox{ id: sbs.ID, controller: sbs.c, containerID: sbs.Cid, endpoints: epHeap{}, populatedEndpoints: map[string]struct{}{}, dbIndex: sbs.dbIndex, isStub: true, dbExists: true, extDNS: sbs.ExtDNS, } msg := " for cleanup" create := true isRestore := false if val, ok := activeSandboxes[sb.ID()]; ok { msg = "" sb.isStub = false isRestore = true opts := val.([]SandboxOption) sb.processOptions(opts...) sb.restorePath() create = !sb.config.useDefaultSandBox heap.Init(&sb.endpoints) } sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore) if err != nil { logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err) continue } c.Lock() c.sandboxes[sb.id] = sb c.Unlock() for _, eps := range sbs.Eps { n, err := c.getNetworkFromStore(eps.Nid) var ep *endpoint if err != nil { logrus.Errorf("getNetworkFromStore for nid %s failed while trying to build sandbox for cleanup: %v", eps.Nid, err) n = &network{id: eps.Nid, ctrlr: c, drvOnce: &sync.Once{}, persist: true} ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID} } else { ep, err = n.getEndpointFromStore(eps.Eid) if err != nil { logrus.Errorf("getEndpointFromStore for eid %s failed while trying to build sandbox for cleanup: %v", eps.Eid, err) ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID} } } if _, ok := activeSandboxes[sb.ID()]; ok && err != nil { logrus.Errorf("failed to restore endpoint %s in %s for container %s due to %v", eps.Eid, eps.Nid, sb.ContainerID(), err) continue } heap.Push(&sb.endpoints, ep) } if _, ok := activeSandboxes[sb.ID()]; !ok { logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID) if err := sb.delete(true); err != nil { logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) } continue } // reconstruct osl sandbox field if !sb.config.useDefaultSandBox { if err := sb.restoreOslSandbox(); err != nil { logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID()) continue } } else { c.sboxOnce.Do(func() { c.defOsSbox = sb.osSbox }) } for _, ep := range sb.endpoints { // Watch for service records if !c.isAgent() { c.watchSvcRecord(ep) } } } }
func (c *controller) sandboxCleanup() { store := c.getStore(datastore.LocalScope) if store == nil { logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes") return } kvol, err := store.List(datastore.Key(sandboxPrefix), &sbState{c: c}) if err != nil && err != datastore.ErrKeyNotFound { logrus.Errorf("failed to get sandboxes for scope %s: %v", store.Scope(), err) return } // It's normal for no sandboxes to be found. Just bail out. if err == datastore.ErrKeyNotFound { return } for _, kvo := range kvol { sbs := kvo.(*sbState) logrus.Printf("sandboxcleanup sbs = %+v", sbs) sb := &sandbox{ id: sbs.ID, controller: sbs.c, containerID: sbs.Cid, endpoints: epHeap{}, epPriority: map[string]int{}, dbIndex: sbs.dbIndex, dbExists: true, } sb.osSbox, err = osl.NewSandbox(sb.Key(), true) if err != nil { logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err) continue } for _, eps := range sbs.Eps { n, err := c.getNetworkFromStore(eps.Nid) if err != nil { logrus.Errorf("getNetworkFromStore for nid %s failed while trying to build sandbox for cleanup: %v", eps.Nid, err) continue } ep, err := n.getEndpointFromStore(eps.Eid) if err != nil { logrus.Errorf("getEndpointFromStore for eid %s failed while trying to build sandbox for cleanup: %v", eps.Eid, err) continue } heap.Push(&sb.endpoints, ep) } c.Lock() c.sandboxes[sb.id] = sb c.Unlock() if err := sb.Delete(); err != nil { logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) } } }