func (s *S) TestClusterHookBeforeCreateContainerIgnoresExistingError(c *check.C) { _, err := InitializeBS() c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() var buf safe.Buffer err = RecreateContainers(p, &buf) c.Assert(err, check.IsNil) nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) hook := ClusterHook{Provisioner: p} err = hook.RunClusterHook(cluster.HookEventBeforeContainerCreate, &nodes[0]) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) client, err := nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, BsDefaultName) client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err = client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, BsDefaultName) }
func (s *S) TestClusterHookBeforeCreateContainerIgnoresExistingError(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() err = RecreateContainers(p) c.Assert(err, check.IsNil) nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) hook := ClusterHook{Provisioner: p} err = hook.BeforeCreateContainer(nodes[0]) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) client, err := nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err = client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") }
func (s *S) TestEnsureContainersStartedOnlyChild(c *check.C) { err := nodecontainer.AddNewContainer("p1", &nodecontainer.NodeContainerConfig{ Name: "c1", Config: docker.Config{ Image: "myregistry/tsuru/bs", }, }) c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() clust := p.Cluster() nodes, err := clust.Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) nodes[0].Metadata["pool"] = "p1" _, err = p.Cluster().UpdateNode(nodes[0]) c.Assert(err, check.IsNil) nodes[1].Metadata["pool"] = "p2" _, err = p.Cluster().UpdateNode(nodes[1]) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) client, err := docker.NewClient(nodes[0].Address) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) client2, err := docker.NewClient(nodes[1].Address) c.Assert(err, check.IsNil) containers2, err := client2.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers2, check.HasLen, 0) }
func (s *S) TestRunContainerHealerCreatedContainerNoProcess(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() app := provisiontest.NewFakeApp("myapp", "python", 2) containers, err := p.StartContainers(dockertest.StartContainersArgs{ Endpoint: p.Servers()[0].URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) notToMove := containers[1] notToMove.MongoID = bson.NewObjectIdWithTime(time.Now().Add(-2 * time.Minute)) notToMove.ProcessName = "" p.PrepareListResult([]container.Container{containers[0], notToMove}, nil) healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() movings := p.Movings() c.Assert(movings, check.IsNil) healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() n, err := healingColl.Count() c.Assert(err, check.IsNil) c.Assert(n, check.Equals, 0) }
func (s *S) TestRunContainerHealerDoesntHealWhenContainerIsRestarting(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() node1 := p.Servers()[0] app := provisiontest.NewFakeApp("myapp", "python", 0) cont, err := p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 1}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) node1.MutateContainer(cont[0].ID, docker.State{Running: false, Restarting: true}) toMoveCont := cont[0] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute) p.PrepareListResult([]container.Container{toMoveCont}, nil) healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() var events []HealingEvent err = healingColl.Find(nil).All(&events) c.Assert(err, check.IsNil) c.Assert(events, check.HasLen, 0) }
func (s *S) TestRecreateBsContainers(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() err = RecreateContainers(p) c.Assert(err, check.IsNil) nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) client, err := nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err = client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") }
func (s *S) TestRunContainerHealerRemovedFromDB(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() node1 := p.Servers()[0] app := newFakeAppInDB("myapp", "python", 0) _, err = p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 1}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) containers := p.AllContainers() c.Assert(err, check.IsNil) p.DeleteContainer(containers[0].ID) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[0] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()}) err = healer.healContainerIfNeeded(toMoveCont) c.Assert(err, check.IsNil) }
func (s *S) TestRecreateBsContainersErrorInSomeContainers(c *check.C) { _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) servers := p.Servers() servers[0].PrepareFailure("failure-create", "/containers/create") defer servers[1].ResetFailure("failure-create") var buf safe.Buffer err = recreateContainers(p, &buf) c.Assert(err, check.ErrorMatches, `(?s)API error \(400\): failure-create.*failed to create container in .* \[.*\].*`) sort.Sort(cluster.NodeList(nodes)) client, err := nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 0) client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) }
func (s *S) TestRunContainerHealerWithError(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() node1 := p.Servers()[0] app := newFakeAppInDB("myapp", "python", 0) _, err = p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) containers := p.AllContainers() c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL())) c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL())) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) p.FailMove( errors.New("cannot move container"), errors.New("cannot move container"), errors.New("cannot move container"), ) healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() containers = p.AllContainers() c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) hosts := []string{containers[0].HostAddr, containers[1].HostAddr} c.Assert(hosts[0], check.Equals, net.URLToHost(node1.URL())) c.Assert(hosts[1], check.Equals, net.URLToHost(node1.URL())) c.Assert(eventtest.EventDesc{ Target: event.Target{Type: "container", Value: toMoveCont.ID}, Kind: "healer", StartCustomData: map[string]interface{}{ "hostaddr": "127.0.0.1", "id": toMoveCont.ID, }, ErrorMatches: `.*Error trying to heal containers.*`, EndCustomData: map[string]interface{}{ "hostaddr": "", }, }, eventtest.HasEvent) }
func (s *S) TestRunContainerHealerAlreadyHealed(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() node1 := p.Servers()[0] app := provisiontest.NewFakeApp("myapp", "python", 0) _, err = p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) containers := p.AllContainers() c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL())) c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL())) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) node1.PrepareFailure("createError", "/containers/create") healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()}) healer.healContainerIfNeeded(toMoveCont) healer.healContainerIfNeeded(toMoveCont) expected := dockertest.ContainerMoving{ ContainerID: toMoveCont.ID, HostFrom: toMoveCont.HostAddr, HostTo: "", } movings := p.Movings() c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected}) healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() var events []HealingEvent err = healingColl.Find(nil).All(&events) c.Assert(err, check.IsNil) c.Assert(events, check.HasLen, 1) c.Assert(events[0].Action, check.Equals, "container-healing") c.Assert(events[0].StartTime.IsZero(), check.Equals, false) c.Assert(events[0].EndTime.IsZero(), check.Equals, false) c.Assert(events[0].Error, check.Equals, "") c.Assert(events[0].Successful, check.Equals, true) c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1") c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1") }
func (s *S) TestRunContainerHealerShutdown(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() node1 := p.Servers()[0] app := provisiontest.NewFakeApp("myapp", "python", 0) _, err = p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) containers := p.AllContainers() c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL())) c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL())) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) node1.PrepareFailure("createError", "/containers/create") healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Done: make(chan bool), Locker: dockertest.NewFakeLocker(), }) ch := make(chan bool) go func() { defer close(ch) healer.RunContainerHealer() }() healer.Shutdown() <-ch expected := dockertest.ContainerMoving{ ContainerID: toMoveCont.ID, HostFrom: toMoveCont.HostAddr, HostTo: "", } movings := p.Movings() c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected}) }
func (s *S) TestRunContainerHealerCreatedContainer(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() app := provisiontest.NewFakeApp("myapp", "python", 2) node1 := p.Servers()[0] containers, err := p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.MongoID = bson.NewObjectIdWithTime(time.Now().Add(-2 * time.Minute)) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) node1.PrepareFailure("createError", "/containers/create") healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() expected := []dockertest.ContainerMoving{ { ContainerID: toMoveCont.ID, HostFrom: toMoveCont.HostAddr, HostTo: "", }, } movings := p.Movings() c.Assert(movings, check.DeepEquals, expected) healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() var events []HealingEvent err = healingColl.Find(nil).All(&events) c.Assert(err, check.IsNil) c.Assert(events, check.HasLen, 1) c.Assert(events[0].Action, check.Equals, "container-healing") c.Assert(events[0].StartTime.IsZero(), check.Equals, false) c.Assert(events[0].EndTime.IsZero(), check.Equals, false) c.Assert(events[0].Error, check.Equals, "") c.Assert(events[0].Successful, check.Equals, true) c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1") c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1") }
func (s *S) TestRunContainerHealerAlreadyHealed(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() node1 := p.Servers()[0] app := newFakeAppInDB("myapp", "python", 0) _, err = p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) containers := p.AllContainers() c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL())) c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL())) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) node1.PrepareFailure("createError", "/containers/create") healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()}) err = healer.healContainerIfNeeded(toMoveCont) c.Assert(err, check.IsNil) err = healer.healContainerIfNeeded(toMoveCont) c.Assert(err, check.IsNil) expected := dockertest.ContainerMoving{ ContainerID: toMoveCont.ID, HostFrom: toMoveCont.HostAddr, HostTo: "", } movings := p.Movings() c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected}) c.Assert(eventtest.EventDesc{ Target: event.Target{Type: "container", Value: toMoveCont.ID}, Kind: "healer", StartCustomData: map[string]interface{}{ "hostaddr": "127.0.0.1", "id": toMoveCont.ID, }, EndCustomData: map[string]interface{}{ "hostaddr": "127.0.0.1", "id": bson.M{"$ne": ""}, }, }, eventtest.HasEvent) }
func (s *S) TestRunContainerHealerStoppedContainer(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() app := newFakeAppInDB("myapp", "python", 2) node1 := p.Servers()[0] containers, err := p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Dead: true}) toMoveCont := containers[1] err = toMoveCont.SetStatus(p, provision.StatusStopped, false) c.Assert(err, check.IsNil) toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) node1.PrepareFailure("createError", "/containers/create") healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() expected := []dockertest.ContainerMoving{ { ContainerID: toMoveCont.ID, HostFrom: toMoveCont.HostAddr, HostTo: "", }, } movings := p.Movings() c.Assert(movings, check.DeepEquals, expected) c.Assert(eventtest.EventDesc{ Target: event.Target{Type: "container", Value: toMoveCont.ID}, Kind: "healer", StartCustomData: map[string]interface{}{ "hostaddr": "127.0.0.1", "id": toMoveCont.ID, }, EndCustomData: map[string]interface{}{ "hostaddr": "127.0.0.1", "id": bson.M{"$ne": ""}, }, }, eventtest.HasEvent) }
func (s *S) TestListUnresponsiveContainersNoHostPort(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() var result []container.Container coll := p.Collection() defer coll.Close() now := time.Now().UTC() coll.Insert( container.Container{ID: "c1", AppName: "app_time_test", LastSuccessStatusUpdate: now.Add(-10 * time.Minute)}, ) defer coll.RemoveAll(bson.M{"appname": "app_time_test"}) result, err = listUnresponsiveContainers(p, 3*time.Minute) c.Assert(err, check.IsNil) c.Assert(result, check.HasLen, 0) }
func (s *S) TestRemoveNamedContainers(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) for i, n := range nodes { n.Metadata["pool"] = fmt.Sprintf("p-%d", i) _, err = p.Cluster().UpdateNode(n) c.Assert(err, check.IsNil) } server := p.Servers()[0] var paths []string server.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { paths = append(paths, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery) server.DefaultHandler().ServeHTTP(w, r) })) server2 := p.Servers()[1] var paths2 []string server2.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { paths2 = append(paths2, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery) server2.DefaultHandler().ServeHTTP(w, r) })) err = ensureContainersStarted(p, ioutil.Discard, true, nil) c.Assert(err, check.IsNil) paths = nil paths2 = nil expectedPaths := []string{"POST /containers/big-sibling/stop?t=10", "DELETE /containers/big-sibling?force=1"} err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "") c.Assert(err, check.IsNil) c.Assert(paths, check.DeepEquals, expectedPaths) c.Assert(paths2, check.DeepEquals, expectedPaths) err = ensureContainersStarted(p, ioutil.Discard, true, nil) c.Assert(err, check.IsNil) paths = nil paths2 = nil err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "p-1") c.Assert(err, check.IsNil) c.Assert(paths, check.DeepEquals, []string(nil)) c.Assert(paths2, check.DeepEquals, expectedPaths) }
func (s *S) TestListUnresponsiveContainers(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() var result []container.Container coll := p.Collection() defer coll.Close() now := time.Now().UTC() coll.Insert( container.Container{ID: "c1", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now, HostPort: "80"}, container.Container{ID: "c2", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now.Add(-1 * time.Minute), HostPort: "80"}, container.Container{ID: "c3", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80"}, ) defer coll.RemoveAll(bson.M{"appname": "app_time_test"}) result, err = listUnresponsiveContainers(p, 3*time.Minute) c.Assert(err, check.IsNil) c.Assert(len(result), check.Equals, 1) c.Assert(result[0].ID, check.Equals, "c3") }
func (s *S) TestRecreateBsContainers(c *check.C) { _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() var buf safe.Buffer err = recreateContainers(p, &buf) c.Assert(err, check.IsNil) nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) client, err := nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err = client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) // It runs in parallel, so we check both ordering output1 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s [] relaunching node container "big-sibling" in the node %s [] `, nodes[0].Address, nodes[1].Address) output2 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s [] relaunching node container "big-sibling" in the node %s [] `, nodes[1].Address, nodes[0].Address) if got := buf.String(); got != output1 && got != output2 { c.Errorf("Wrong output:\n%s", got) } }
func (s *S) TestRunContainerHealerThrottled(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() node1 := p.Servers()[0] app := newFakeAppInDB("myapp", "python", 0) _, err = p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) containers := p.AllContainers() c.Assert(containers, check.HasLen, 2) c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL())) c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL())) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute) for i := 0; i < 3; i++ { var evt *event.Event evt, err = event.NewInternal(&event.Opts{ Target: event.Target{Type: "container", Value: toMoveCont.ID}, InternalKind: "healer", CustomData: toMoveCont, Allowed: event.Allowed(permission.PermAppReadEvents), }) c.Assert(err, check.IsNil) err = evt.DoneCustomData(nil, nil) c.Assert(err, check.IsNil) } healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()}) err = healer.healContainerIfNeeded(toMoveCont) c.Assert(err, check.ErrorMatches, "Error trying to insert container healing event, healing aborted: event throttled, limit for healer on container \".*?\" is 3 every 5m0s") }
func (s *S) TestListUnresponsiveContainersIncludeStopped(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() var result []container.Container coll := p.Collection() defer coll.Close() now := time.Now().UTC() coll.Insert( container.Container{ID: "c1", AppName: "app_time_test", LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80", Status: provision.StatusStopped.String()}, container.Container{ID: "c2", AppName: "app_time_test", LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80", Status: provision.StatusStarted.String()}, ) defer coll.RemoveAll(bson.M{"appname": "app_time_test"}) result, err = listUnresponsiveContainers(p, 3*time.Minute) c.Assert(err, check.IsNil) c.Assert(result, check.HasLen, 2) ids := []string{result[0].ID, result[1].ID} sort.Strings(ids) c.Assert(ids, check.DeepEquals, []string{"c1", "c2"}) }
func (s *S) TestClusterHookBeforeCreateContainerStartsStopped(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() var buf safe.Buffer err = RecreateContainers(p, &buf) c.Assert(err, check.IsNil) nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) client, err := nodes[0].Client() c.Assert(err, check.IsNil) err = client.StopContainer("big-sibling", 1) c.Assert(err, check.IsNil) contData, err := client.InspectContainer("big-sibling") c.Assert(err, check.IsNil) c.Assert(contData.State.Running, check.Equals, false) hook := ClusterHook{Provisioner: p} err = hook.BeforeCreateContainer(nodes[0]) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) client, err = nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") c.Assert(container.State.Running, check.Equals, true) client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err = client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") c.Assert(container.State.Running, check.Equals, true) }
func (s *S) TestRunContainerHealerStoppedContainerAlreadyStopped(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() app := newFakeAppInDB("myapp", "python", 2) node1 := p.Servers()[0] containers, err := p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] err = toMoveCont.SetStatus(p, provision.StatusStopped, false) c.Assert(err, check.IsNil) toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) node1.PrepareFailure("createError", "/containers/create") healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() movings := p.Movings() c.Assert(movings, check.IsNil) c.Assert(eventtest.EventDesc{ IsEmpty: true, }, eventtest.HasEvent) conts, err := p.ListContainers(bson.M{"id": toMoveCont.ID}) c.Assert(err, check.IsNil) c.Assert(conts, check.HasLen, 1) c.Assert(conts[0].Status, check.Equals, provision.StatusStopped.String()) c.Assert(time.Since(conts[0].LastSuccessStatusUpdate) < time.Minute, check.Equals, true) }
func (s *S) TestHealContainer(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() containers := []container.Container{ {ID: "cont1", AppName: "app1"}, {ID: "cont2", AppName: "app1"}, {ID: "cont3", AppName: "app2"}, } p.SetContainers("localhost", containers) locker := dockertest.NewFakeLocker() locked := locker.Lock(containers[0].AppName) c.Assert(locked, check.Equals, true) defer locker.Unlock(containers[0].AppName) healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: locker}) _, err = healer.healContainer(containers[0]) c.Assert(err, check.IsNil) expected := []container.Container{ {ID: "cont1-recreated", HostAddr: "localhost", AppName: "app1"}, {ID: "cont2", AppName: "app1", HostAddr: "localhost"}, {ID: "cont3", AppName: "app2", HostAddr: "localhost"}, } c.Assert(p.Containers("localhost"), check.DeepEquals, expected) }
func (s *S) TestRunContainerHealer(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() app := provisiontest.NewFakeApp("myapp", "python", 2) node1 := p.Servers()[0] containers, err := p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) node1.PrepareFailure("createError", "/containers/create") healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() expected := []dockertest.ContainerMoving{ { ContainerID: toMoveCont.ID, HostFrom: toMoveCont.HostAddr, HostTo: "", }, } movings := p.Movings() c.Assert(movings, check.DeepEquals, expected) queries := p.Queries() c.Assert(queries, check.HasLen, 1) queryTime := queries[0]["lastsuccessstatusupdate"].(bson.M)["$lt"].(time.Time) delete(queries[0], "lastsuccessstatusupdate") c.Assert(time.Now().UTC().Add(-1*time.Minute).Sub(queryTime) < time.Second, check.Equals, true) c.Assert(queries, check.DeepEquals, []bson.M{{ "$or": []bson.M{ {"hostport": bson.M{"$ne": ""}}, {"processname": bson.M{"$ne": ""}}, }, "status": bson.M{"$nin": []string{ provision.StatusStopped.String(), provision.StatusBuilding.String(), provision.StatusAsleep.String(), }}, }}) healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() var events []HealingEvent err = healingColl.Find(nil).All(&events) c.Assert(err, check.IsNil) c.Assert(events, check.HasLen, 1) c.Assert(events[0].Action, check.Equals, "container-healing") c.Assert(events[0].StartTime.IsZero(), check.Equals, false) c.Assert(events[0].EndTime.IsZero(), check.Equals, false) c.Assert(events[0].Error, check.Equals, "") c.Assert(events[0].Successful, check.Equals, true) c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1") c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1") }
func (s *S) TestRunContainerHealerWithError(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() node1 := p.Servers()[0] app := provisiontest.NewFakeApp("myapp", "python", 0) _, err = p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) containers := p.AllContainers() c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL())) c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL())) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) p.FailMove( errors.New("cannot move container"), errors.New("cannot move container"), errors.New("cannot move container"), ) healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() containers = p.AllContainers() c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) hosts := []string{containers[0].HostAddr, containers[1].HostAddr} c.Assert(hosts[0], check.Equals, net.URLToHost(node1.URL())) c.Assert(hosts[1], check.Equals, net.URLToHost(node1.URL())) healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() var events []HealingEvent err = healingColl.Find(nil).All(&events) c.Assert(err, check.IsNil) c.Assert(events, check.HasLen, 1) c.Assert(events[0].Action, check.Equals, "container-healing") c.Assert(events[0].StartTime.IsZero(), check.Equals, false) c.Assert(events[0].EndTime.IsZero(), check.Equals, false) c.Assert(events[0].Error, check.Matches, "(?s).*Error trying to heal containers.*") c.Assert(events[0].Successful, check.Equals, false) c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1") c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "") }
func (s *S) TestRunContainerHealer(c *check.C) { p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() app := newFakeAppInDB("myapp", "python", 2) node1 := p.Servers()[0] containers, err := p.StartContainers(dockertest.StartContainersArgs{ Endpoint: node1.URL(), App: app, Amount: map[string]int{"web": 2}, Image: "tsuru/python", PullImage: true, }) c.Assert(err, check.IsNil) node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false}) node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false}) toMoveCont := containers[1] toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute) p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil) node1.PrepareFailure("createError", "/containers/create") healer := NewContainerHealer(ContainerHealerArgs{ Provisioner: p, MaxUnresponsiveTime: time.Minute, Locker: dockertest.NewFakeLocker(), }) healer.runContainerHealerOnce() expected := []dockertest.ContainerMoving{ { ContainerID: toMoveCont.ID, HostFrom: toMoveCont.HostAddr, HostTo: "", }, } movings := p.Movings() c.Assert(movings, check.DeepEquals, expected) queries := p.Queries() c.Assert(queries, check.HasLen, 1) queryTime := queries[0]["lastsuccessstatusupdate"].(bson.M)["$lt"].(time.Time) delete(queries[0], "lastsuccessstatusupdate") c.Assert(time.Now().UTC().Add(-1*time.Minute).Sub(queryTime) < time.Second, check.Equals, true) c.Assert(queries, check.DeepEquals, []bson.M{{ "id": bson.M{"$ne": ""}, "appname": bson.M{"$ne": ""}, "$or": []bson.M{ {"hostport": bson.M{"$ne": ""}}, {"processname": bson.M{"$ne": ""}}, }, "status": bson.M{"$nin": []string{ provision.StatusBuilding.String(), provision.StatusAsleep.String(), }}, }}) c.Assert(eventtest.EventDesc{ Target: event.Target{Type: "container", Value: toMoveCont.ID}, Kind: "healer", StartCustomData: map[string]interface{}{ "hostaddr": "127.0.0.1", "id": toMoveCont.ID, }, EndCustomData: map[string]interface{}{ "hostaddr": "127.0.0.1", "id": bson.M{"$ne": ""}, }, }, eventtest.HasEvent) }
func (s *S) TestEnsureContainersStarted(c *check.C) { c1 := NodeContainerConfig{ Name: "bs", Config: docker.Config{ Image: "bsimg", Env: []string{ "A=1", "B=2", }, }, HostConfig: docker.HostConfig{ RestartPolicy: docker.AlwaysRestart(), Privileged: true, Binds: []string{"/xyz:/abc:rw"}, }, } err := AddNewContainer("", &c1) c.Assert(err, check.IsNil) c2 := c1 c2.Name = "sysdig" c2.Config.Image = "sysdigimg" c2.Config.Env = []string{"X=Z"} err = AddNewContainer("", &c2) c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) var createBodies []string var names []string var mut sync.Mutex server := p.Servers()[0] server.CustomHandler("/containers/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mut.Lock() defer mut.Unlock() data, _ := ioutil.ReadAll(r.Body) createBodies = append(createBodies, string(data)) names = append(names, r.URL.Query().Get("name")) r.Body = ioutil.NopCloser(bytes.NewBuffer(data)) server.DefaultHandler().ServeHTTP(w, r) })) defer p.Destroy() buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true) c.Assert(err, check.IsNil) parts := strings.Split(buf.String(), "\n") c.Assert(parts, check.HasLen, 5) sort.Strings(parts) c.Assert(parts[1], check.Matches, `relaunching node container "bs" in the node http://127.0.0.1:\d+/ \[\]`) c.Assert(parts[2], check.Matches, `relaunching node container "bs" in the node http://localhost:\d+/ \[\]`) c.Assert(parts[3], check.Matches, `relaunching node container "sysdig" in the node http://127.0.0.1:\d+/ \[\]`) c.Assert(parts[4], check.Matches, `relaunching node container "sysdig" in the node http://localhost:\d+/ \[\]`) c.Assert(createBodies, check.HasLen, 2) c.Assert(names, check.HasLen, 2) sort.Strings(names) c.Assert(names, check.DeepEquals, []string{"bs", "sysdig"}) sort.Strings(createBodies) result := make([]struct { docker.Config HostConfig docker.HostConfig }, 2) err = json.Unmarshal([]byte(createBodies[0]), &result[0]) c.Assert(err, check.IsNil) err = json.Unmarshal([]byte(createBodies[1]), &result[1]) c.Assert(err, check.IsNil) c.Assert(result, check.DeepEquals, []struct { docker.Config HostConfig docker.HostConfig }{ { Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "A=1", "B=2"}, Image: "bsimg"}, HostConfig: docker.HostConfig{ Binds: []string{"/xyz:/abc:rw"}, Privileged: true, RestartPolicy: docker.RestartPolicy{Name: "always"}, LogConfig: docker.LogConfig{}, }, }, { Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "X=Z"}, Image: "sysdigimg"}, HostConfig: docker.HostConfig{ Binds: []string{"/xyz:/abc:rw"}, Privileged: true, RestartPolicy: docker.RestartPolicy{Name: "always"}, LogConfig: docker.LogConfig{}, }, }, }) conf := configFor("bs") var result1 NodeContainerConfig err = conf.Load("", &result1) c.Assert(err, check.IsNil) c.Assert(result1.PinnedImage, check.Equals, "") client, err := docker.NewClient(p.Servers()[0].URL()) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) client, err = docker.NewClient(p.Servers()[1].URL()) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) }