func (s *S) TestClusterHookBeforeCreateContainerIgnoresExistingError(c *check.C) {
_, err := InitializeBS()
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
var buf safe.Buffer
err = RecreateContainers(p, &buf)
c.Assert(err, check.IsNil)
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
hook := ClusterHook{Provisioner: p}
err = hook.RunClusterHook(cluster.HookEventBeforeContainerCreate, &nodes[0])
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, BsDefaultName)
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err = client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, BsDefaultName)
}
func (s *S) TestClusterHookBeforeCreateContainerIgnoresExistingError(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
err = RecreateContainers(p)
c.Assert(err, check.IsNil)
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
hook := ClusterHook{Provisioner: p}
err = hook.BeforeCreateContainer(nodes[0])
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err = client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
}
func (s *S) TestEnsureContainersStartedOnlyChild(c *check.C) {
err := nodecontainer.AddNewContainer("p1", &nodecontainer.NodeContainerConfig{
Name: "c1",
Config: docker.Config{
Image: "myregistry/tsuru/bs",
},
})
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
clust := p.Cluster()
nodes, err := clust.Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
nodes[0].Metadata["pool"] = "p1"
_, err = p.Cluster().UpdateNode(nodes[0])
c.Assert(err, check.IsNil)
nodes[1].Metadata["pool"] = "p2"
_, err = p.Cluster().UpdateNode(nodes[1])
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
client, err := docker.NewClient(nodes[0].Address)
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
client2, err := docker.NewClient(nodes[1].Address)
c.Assert(err, check.IsNil)
containers2, err := client2.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers2, check.HasLen, 0)
}
func (s *S) TestRunContainerHealerCreatedContainerNoProcess(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := provisiontest.NewFakeApp("myapp", "python", 2)
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: p.Servers()[0].URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
notToMove := containers[1]
notToMove.MongoID = bson.NewObjectIdWithTime(time.Now().Add(-2 * time.Minute))
notToMove.ProcessName = ""
p.PrepareListResult([]container.Container{containers[0], notToMove}, nil)
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
movings := p.Movings()
c.Assert(movings, check.IsNil)
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
n, err := healingColl.Count()
c.Assert(err, check.IsNil)
c.Assert(n, check.Equals, 0)
}
func (s *S) TestRunContainerHealerDoesntHealWhenContainerIsRestarting(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := provisiontest.NewFakeApp("myapp", "python", 0)
cont, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 1},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(cont[0].ID, docker.State{Running: false, Restarting: true})
toMoveCont := cont[0]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
p.PrepareListResult([]container.Container{toMoveCont}, nil)
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 0)
}
func (s *S) TestRecreateBsContainers(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
err = RecreateContainers(p)
c.Assert(err, check.IsNil)
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err = client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
}
func (s *S) TestRunContainerHealerRemovedFromDB(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := newFakeAppInDB("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 1},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
p.DeleteContainer(containers[0].ID)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[0]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.IsNil)
}
func (s *S) TestRecreateBsContainersErrorInSomeContainers(c *check.C) {
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
servers := p.Servers()
servers[0].PrepareFailure("failure-create", "/containers/create")
defer servers[1].ResetFailure("failure-create")
var buf safe.Buffer
err = recreateContainers(p, &buf)
c.Assert(err, check.ErrorMatches, `(?s)API error \(400\): failure-create.*failed to create container in .* \[.*\].*`)
sort.Sort(cluster.NodeList(nodes))
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 0)
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
}
func (s *S) TestRunContainerHealerWithError(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := newFakeAppInDB("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
p.FailMove(
errors.New("cannot move container"),
errors.New("cannot move container"),
errors.New("cannot move container"),
)
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
containers = p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
hosts := []string{containers[0].HostAddr, containers[1].HostAddr}
c.Assert(hosts[0], check.Equals, net.URLToHost(node1.URL()))
c.Assert(hosts[1], check.Equals, net.URLToHost(node1.URL()))
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
Kind: "healer",
StartCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": toMoveCont.ID,
},
ErrorMatches: `.*Error trying to heal containers.*`,
EndCustomData: map[string]interface{}{
"hostaddr": "",
},
}, eventtest.HasEvent)
}
func (s *S) TestRunContainerHealerAlreadyHealed(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := provisiontest.NewFakeApp("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
healer.healContainerIfNeeded(toMoveCont)
healer.healContainerIfNeeded(toMoveCont)
expected := dockertest.ContainerMoving{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "container-healing")
c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
func (s *S) TestRunContainerHealerShutdown(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := provisiontest.NewFakeApp("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Done: make(chan bool),
Locker: dockertest.NewFakeLocker(),
})
ch := make(chan bool)
go func() {
defer close(ch)
healer.RunContainerHealer()
}()
healer.Shutdown()
<-ch
expected := dockertest.ContainerMoving{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})
}
func (s *S) TestRunContainerHealerCreatedContainer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := provisiontest.NewFakeApp("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.MongoID = bson.NewObjectIdWithTime(time.Now().Add(-2 * time.Minute))
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
expected := []dockertest.ContainerMoving{
{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
},
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, expected)
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "container-healing")
c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
func (s *S) TestRunContainerHealerAlreadyHealed(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := newFakeAppInDB("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.IsNil)
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.IsNil)
expected := dockertest.ContainerMoving{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
Kind: "healer",
StartCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": toMoveCont.ID,
},
EndCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": bson.M{"$ne": ""},
},
}, eventtest.HasEvent)
}
func (s *S) TestRunContainerHealerStoppedContainer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := newFakeAppInDB("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Dead: true})
toMoveCont := containers[1]
err = toMoveCont.SetStatus(p, provision.StatusStopped, false)
c.Assert(err, check.IsNil)
toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
expected := []dockertest.ContainerMoving{
{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
},
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, expected)
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
Kind: "healer",
StartCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": toMoveCont.ID,
},
EndCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": bson.M{"$ne": ""},
},
}, eventtest.HasEvent)
}
func (s *S) TestListUnresponsiveContainersNoHostPort(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
var result []container.Container
coll := p.Collection()
defer coll.Close()
now := time.Now().UTC()
coll.Insert(
container.Container{ID: "c1", AppName: "app_time_test", LastSuccessStatusUpdate: now.Add(-10 * time.Minute)},
)
defer coll.RemoveAll(bson.M{"appname": "app_time_test"})
result, err = listUnresponsiveContainers(p, 3*time.Minute)
c.Assert(err, check.IsNil)
c.Assert(result, check.HasLen, 0)
}
func (s *S) TestRemoveNamedContainers(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
for i, n := range nodes {
n.Metadata["pool"] = fmt.Sprintf("p-%d", i)
_, err = p.Cluster().UpdateNode(n)
c.Assert(err, check.IsNil)
}
server := p.Servers()[0]
var paths []string
server.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
paths = append(paths, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery)
server.DefaultHandler().ServeHTTP(w, r)
}))
server2 := p.Servers()[1]
var paths2 []string
server2.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
paths2 = append(paths2, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery)
server2.DefaultHandler().ServeHTTP(w, r)
}))
err = ensureContainersStarted(p, ioutil.Discard, true, nil)
c.Assert(err, check.IsNil)
paths = nil
paths2 = nil
expectedPaths := []string{"POST /containers/big-sibling/stop?t=10", "DELETE /containers/big-sibling?force=1"}
err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "")
c.Assert(err, check.IsNil)
c.Assert(paths, check.DeepEquals, expectedPaths)
c.Assert(paths2, check.DeepEquals, expectedPaths)
err = ensureContainersStarted(p, ioutil.Discard, true, nil)
c.Assert(err, check.IsNil)
paths = nil
paths2 = nil
err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "p-1")
c.Assert(err, check.IsNil)
c.Assert(paths, check.DeepEquals, []string(nil))
c.Assert(paths2, check.DeepEquals, expectedPaths)
}
func (s *S) TestListUnresponsiveContainers(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
var result []container.Container
coll := p.Collection()
defer coll.Close()
now := time.Now().UTC()
coll.Insert(
container.Container{ID: "c1", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now, HostPort: "80"},
container.Container{ID: "c2", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now.Add(-1 * time.Minute), HostPort: "80"},
container.Container{ID: "c3", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80"},
)
defer coll.RemoveAll(bson.M{"appname": "app_time_test"})
result, err = listUnresponsiveContainers(p, 3*time.Minute)
c.Assert(err, check.IsNil)
c.Assert(len(result), check.Equals, 1)
c.Assert(result[0].ID, check.Equals, "c3")
}
func (s *S) TestRecreateBsContainers(c *check.C) {
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
var buf safe.Buffer
err = recreateContainers(p, &buf)
c.Assert(err, check.IsNil)
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err = client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
// It runs in parallel, so we check both ordering
output1 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s []
relaunching node container "big-sibling" in the node %s []
`, nodes[0].Address, nodes[1].Address)
output2 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s []
relaunching node container "big-sibling" in the node %s []
`, nodes[1].Address, nodes[0].Address)
if got := buf.String(); got != output1 && got != output2 {
c.Errorf("Wrong output:\n%s", got)
}
}
func (s *S) TestRunContainerHealerThrottled(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := newFakeAppInDB("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
for i := 0; i < 3; i++ {
var evt *event.Event
evt, err = event.NewInternal(&event.Opts{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
InternalKind: "healer",
CustomData: toMoveCont,
Allowed: event.Allowed(permission.PermAppReadEvents),
})
c.Assert(err, check.IsNil)
err = evt.DoneCustomData(nil, nil)
c.Assert(err, check.IsNil)
}
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.ErrorMatches, "Error trying to insert container healing event, healing aborted: event throttled, limit for healer on container \".*?\" is 3 every 5m0s")
}
func (s *S) TestListUnresponsiveContainersIncludeStopped(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
var result []container.Container
coll := p.Collection()
defer coll.Close()
now := time.Now().UTC()
coll.Insert(
container.Container{ID: "c1", AppName: "app_time_test",
LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80", Status: provision.StatusStopped.String()},
container.Container{ID: "c2", AppName: "app_time_test",
LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80", Status: provision.StatusStarted.String()},
)
defer coll.RemoveAll(bson.M{"appname": "app_time_test"})
result, err = listUnresponsiveContainers(p, 3*time.Minute)
c.Assert(err, check.IsNil)
c.Assert(result, check.HasLen, 2)
ids := []string{result[0].ID, result[1].ID}
sort.Strings(ids)
c.Assert(ids, check.DeepEquals, []string{"c1", "c2"})
}
func (s *S) TestClusterHookBeforeCreateContainerStartsStopped(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
var buf safe.Buffer
err = RecreateContainers(p, &buf)
c.Assert(err, check.IsNil)
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
err = client.StopContainer("big-sibling", 1)
c.Assert(err, check.IsNil)
contData, err := client.InspectContainer("big-sibling")
c.Assert(err, check.IsNil)
c.Assert(contData.State.Running, check.Equals, false)
hook := ClusterHook{Provisioner: p}
err = hook.BeforeCreateContainer(nodes[0])
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
client, err = nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
c.Assert(container.State.Running, check.Equals, true)
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err = client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
c.Assert(container.State.Running, check.Equals, true)
}
func (s *S) TestRunContainerHealerStoppedContainerAlreadyStopped(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := newFakeAppInDB("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
err = toMoveCont.SetStatus(p, provision.StatusStopped, false)
c.Assert(err, check.IsNil)
toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
movings := p.Movings()
c.Assert(movings, check.IsNil)
c.Assert(eventtest.EventDesc{
IsEmpty: true,
}, eventtest.HasEvent)
conts, err := p.ListContainers(bson.M{"id": toMoveCont.ID})
c.Assert(err, check.IsNil)
c.Assert(conts, check.HasLen, 1)
c.Assert(conts[0].Status, check.Equals, provision.StatusStopped.String())
c.Assert(time.Since(conts[0].LastSuccessStatusUpdate) < time.Minute, check.Equals, true)
}
func (s *S) TestHealContainer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
containers := []container.Container{
{ID: "cont1", AppName: "app1"},
{ID: "cont2", AppName: "app1"},
{ID: "cont3", AppName: "app2"},
}
p.SetContainers("localhost", containers)
locker := dockertest.NewFakeLocker()
locked := locker.Lock(containers[0].AppName)
c.Assert(locked, check.Equals, true)
defer locker.Unlock(containers[0].AppName)
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: locker})
_, err = healer.healContainer(containers[0])
c.Assert(err, check.IsNil)
expected := []container.Container{
{ID: "cont1-recreated", HostAddr: "localhost", AppName: "app1"},
{ID: "cont2", AppName: "app1", HostAddr: "localhost"},
{ID: "cont3", AppName: "app2", HostAddr: "localhost"},
}
c.Assert(p.Containers("localhost"), check.DeepEquals, expected)
}
func (s *S) TestRunContainerHealer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := provisiontest.NewFakeApp("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
expected := []dockertest.ContainerMoving{
{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
},
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, expected)
queries := p.Queries()
c.Assert(queries, check.HasLen, 1)
queryTime := queries[0]["lastsuccessstatusupdate"].(bson.M)["$lt"].(time.Time)
delete(queries[0], "lastsuccessstatusupdate")
c.Assert(time.Now().UTC().Add(-1*time.Minute).Sub(queryTime) < time.Second, check.Equals, true)
c.Assert(queries, check.DeepEquals, []bson.M{{
"$or": []bson.M{
{"hostport": bson.M{"$ne": ""}},
{"processname": bson.M{"$ne": ""}},
},
"status": bson.M{"$nin": []string{
provision.StatusStopped.String(),
provision.StatusBuilding.String(),
provision.StatusAsleep.String(),
}},
}})
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "container-healing")
c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
func (s *S) TestRunContainerHealerWithError(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := provisiontest.NewFakeApp("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
p.FailMove(
errors.New("cannot move container"),
errors.New("cannot move container"),
errors.New("cannot move container"),
)
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
containers = p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
hosts := []string{containers[0].HostAddr, containers[1].HostAddr}
c.Assert(hosts[0], check.Equals, net.URLToHost(node1.URL()))
c.Assert(hosts[1], check.Equals, net.URLToHost(node1.URL()))
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "container-healing")
c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
c.Assert(events[0].Error, check.Matches, "(?s).*Error trying to heal containers.*")
c.Assert(events[0].Successful, check.Equals, false)
c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "")
}
func (s *S) TestRunContainerHealer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := newFakeAppInDB("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
expected := []dockertest.ContainerMoving{
{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
},
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, expected)
queries := p.Queries()
c.Assert(queries, check.HasLen, 1)
queryTime := queries[0]["lastsuccessstatusupdate"].(bson.M)["$lt"].(time.Time)
delete(queries[0], "lastsuccessstatusupdate")
c.Assert(time.Now().UTC().Add(-1*time.Minute).Sub(queryTime) < time.Second, check.Equals, true)
c.Assert(queries, check.DeepEquals, []bson.M{{
"id": bson.M{"$ne": ""},
"appname": bson.M{"$ne": ""},
"$or": []bson.M{
{"hostport": bson.M{"$ne": ""}},
{"processname": bson.M{"$ne": ""}},
},
"status": bson.M{"$nin": []string{
provision.StatusBuilding.String(),
provision.StatusAsleep.String(),
}},
}})
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
Kind: "healer",
StartCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": toMoveCont.ID,
},
EndCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": bson.M{"$ne": ""},
},
}, eventtest.HasEvent)
}
func (s *S) TestEnsureContainersStarted(c *check.C) {
c1 := NodeContainerConfig{
Name: "bs",
Config: docker.Config{
Image: "bsimg",
Env: []string{
"A=1",
"B=2",
},
},
HostConfig: docker.HostConfig{
RestartPolicy: docker.AlwaysRestart(),
Privileged: true,
Binds: []string{"/xyz:/abc:rw"},
},
}
err := AddNewContainer("", &c1)
c.Assert(err, check.IsNil)
c2 := c1
c2.Name = "sysdig"
c2.Config.Image = "sysdigimg"
c2.Config.Env = []string{"X=Z"}
err = AddNewContainer("", &c2)
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
var createBodies []string
var names []string
var mut sync.Mutex
server := p.Servers()[0]
server.CustomHandler("/containers/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mut.Lock()
defer mut.Unlock()
data, _ := ioutil.ReadAll(r.Body)
createBodies = append(createBodies, string(data))
names = append(names, r.URL.Query().Get("name"))
r.Body = ioutil.NopCloser(bytes.NewBuffer(data))
server.DefaultHandler().ServeHTTP(w, r)
}))
defer p.Destroy()
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true)
c.Assert(err, check.IsNil)
parts := strings.Split(buf.String(), "\n")
c.Assert(parts, check.HasLen, 5)
sort.Strings(parts)
c.Assert(parts[1], check.Matches, `relaunching node container "bs" in the node http://127.0.0.1:\d+/ \[\]`)
c.Assert(parts[2], check.Matches, `relaunching node container "bs" in the node http://localhost:\d+/ \[\]`)
c.Assert(parts[3], check.Matches, `relaunching node container "sysdig" in the node http://127.0.0.1:\d+/ \[\]`)
c.Assert(parts[4], check.Matches, `relaunching node container "sysdig" in the node http://localhost:\d+/ \[\]`)
c.Assert(createBodies, check.HasLen, 2)
c.Assert(names, check.HasLen, 2)
sort.Strings(names)
c.Assert(names, check.DeepEquals, []string{"bs", "sysdig"})
sort.Strings(createBodies)
result := make([]struct {
docker.Config
HostConfig docker.HostConfig
}, 2)
err = json.Unmarshal([]byte(createBodies[0]), &result[0])
c.Assert(err, check.IsNil)
err = json.Unmarshal([]byte(createBodies[1]), &result[1])
c.Assert(err, check.IsNil)
c.Assert(result, check.DeepEquals, []struct {
docker.Config
HostConfig docker.HostConfig
}{
{
Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "A=1", "B=2"}, Image: "bsimg"},
HostConfig: docker.HostConfig{
Binds: []string{"/xyz:/abc:rw"},
Privileged: true,
RestartPolicy: docker.RestartPolicy{Name: "always"},
LogConfig: docker.LogConfig{},
},
},
{
Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "X=Z"}, Image: "sysdigimg"},
HostConfig: docker.HostConfig{
Binds: []string{"/xyz:/abc:rw"},
Privileged: true,
RestartPolicy: docker.RestartPolicy{Name: "always"},
LogConfig: docker.LogConfig{},
},
},
})
conf := configFor("bs")
var result1 NodeContainerConfig
err = conf.Load("", &result1)
c.Assert(err, check.IsNil)
c.Assert(result1.PinnedImage, check.Equals, "")
client, err := docker.NewClient(p.Servers()[0].URL())
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
client, err = docker.NewClient(p.Servers()[1].URL())
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
}