Example #1
0
func (s *S) TestClusterHookBeforeCreateContainerIgnoresExistingError(c *check.C) {
	_, err := InitializeBS()
	c.Assert(err, check.IsNil)
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	var buf safe.Buffer
	err = RecreateContainers(p, &buf)
	c.Assert(err, check.IsNil)
	nodes, err := p.Cluster().Nodes()
	c.Assert(err, check.IsNil)
	hook := ClusterHook{Provisioner: p}
	err = hook.RunClusterHook(cluster.HookEventBeforeContainerCreate, &nodes[0])
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 2)
	client, err := nodes[0].Client()
	c.Assert(err, check.IsNil)
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err := client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, BsDefaultName)
	client, err = nodes[1].Client()
	c.Assert(err, check.IsNil)
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err = client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, BsDefaultName)
}
Example #2
0
func (s *S) TestClusterHookBeforeCreateContainerIgnoresExistingError(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	err = RecreateContainers(p)
	c.Assert(err, check.IsNil)
	nodes, err := p.Cluster().Nodes()
	c.Assert(err, check.IsNil)
	hook := ClusterHook{Provisioner: p}
	err = hook.BeforeCreateContainer(nodes[0])
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 2)
	client, err := nodes[0].Client()
	c.Assert(err, check.IsNil)
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err := client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, "big-sibling")
	client, err = nodes[1].Client()
	c.Assert(err, check.IsNil)
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err = client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, "big-sibling")
}
Example #3
0
func (s *S) TestEnsureContainersStartedOnlyChild(c *check.C) {
	err := nodecontainer.AddNewContainer("p1", &nodecontainer.NodeContainerConfig{
		Name: "c1",
		Config: docker.Config{
			Image: "myregistry/tsuru/bs",
		},
	})
	c.Assert(err, check.IsNil)
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	clust := p.Cluster()
	nodes, err := clust.Nodes()
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 2)
	nodes[0].Metadata["pool"] = "p1"
	_, err = p.Cluster().UpdateNode(nodes[0])
	c.Assert(err, check.IsNil)
	nodes[1].Metadata["pool"] = "p2"
	_, err = p.Cluster().UpdateNode(nodes[1])
	c.Assert(err, check.IsNil)
	buf := safe.NewBuffer(nil)
	err = ensureContainersStarted(p, buf, true, nil)
	c.Assert(err, check.IsNil)
	client, err := docker.NewClient(nodes[0].Address)
	c.Assert(err, check.IsNil)
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	client2, err := docker.NewClient(nodes[1].Address)
	c.Assert(err, check.IsNil)
	containers2, err := client2.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers2, check.HasLen, 0)
}
Example #4
0
func (s *S) TestRunContainerHealerCreatedContainerNoProcess(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	app := provisiontest.NewFakeApp("myapp", "python", 2)
	containers, err := p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  p.Servers()[0].URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	notToMove := containers[1]
	notToMove.MongoID = bson.NewObjectIdWithTime(time.Now().Add(-2 * time.Minute))
	notToMove.ProcessName = ""
	p.PrepareListResult([]container.Container{containers[0], notToMove}, nil)
	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()
	movings := p.Movings()
	c.Assert(movings, check.IsNil)
	healingColl, err := healingCollection()
	c.Assert(err, check.IsNil)
	defer healingColl.Close()
	n, err := healingColl.Count()
	c.Assert(err, check.IsNil)
	c.Assert(n, check.Equals, 0)
}
Example #5
0
func (s *S) TestRunContainerHealerDoesntHealWhenContainerIsRestarting(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	node1 := p.Servers()[0]
	app := provisiontest.NewFakeApp("myapp", "python", 0)
	cont, err := p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 1},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	node1.MutateContainer(cont[0].ID, docker.State{Running: false, Restarting: true})

	toMoveCont := cont[0]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
	p.PrepareListResult([]container.Container{toMoveCont}, nil)

	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()

	healingColl, err := healingCollection()
	c.Assert(err, check.IsNil)
	defer healingColl.Close()
	var events []HealingEvent
	err = healingColl.Find(nil).All(&events)
	c.Assert(err, check.IsNil)
	c.Assert(events, check.HasLen, 0)
}
Example #6
0
func (s *S) TestRecreateBsContainers(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	err = RecreateContainers(p)
	c.Assert(err, check.IsNil)
	nodes, err := p.Cluster().Nodes()
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 2)
	client, err := nodes[0].Client()
	c.Assert(err, check.IsNil)
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err := client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, "big-sibling")
	client, err = nodes[1].Client()
	c.Assert(err, check.IsNil)
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err = client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, "big-sibling")
}
Example #7
0
func (s *S) TestRunContainerHealerRemovedFromDB(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	node1 := p.Servers()[0]
	app := newFakeAppInDB("myapp", "python", 0)
	_, err = p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 1},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	containers := p.AllContainers()
	c.Assert(err, check.IsNil)
	p.DeleteContainer(containers[0].ID)
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	toMoveCont := containers[0]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
	err = healer.healContainerIfNeeded(toMoveCont)
	c.Assert(err, check.IsNil)
}
Example #8
0
func (s *S) TestRecreateBsContainersErrorInSomeContainers(c *check.C) {
	_, err := nodecontainer.InitializeBS()
	c.Assert(err, check.IsNil)
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	nodes, err := p.Cluster().Nodes()
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 2)
	servers := p.Servers()
	servers[0].PrepareFailure("failure-create", "/containers/create")
	defer servers[1].ResetFailure("failure-create")
	var buf safe.Buffer
	err = recreateContainers(p, &buf)
	c.Assert(err, check.ErrorMatches, `(?s)API error \(400\): failure-create.*failed to create container in .* \[.*\].*`)
	sort.Sort(cluster.NodeList(nodes))
	client, err := nodes[0].Client()
	c.Assert(err, check.IsNil)
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 0)
	client, err = nodes[1].Client()
	c.Assert(err, check.IsNil)
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err := client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
}
Example #9
0
func (s *S) TestRunContainerHealerWithError(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	node1 := p.Servers()[0]
	app := newFakeAppInDB("myapp", "python", 0)
	_, err = p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)

	containers := p.AllContainers()
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
	c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})

	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	p.FailMove(
		errors.New("cannot move container"),
		errors.New("cannot move container"),
		errors.New("cannot move container"),
	)

	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()

	containers = p.AllContainers()
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
	hosts := []string{containers[0].HostAddr, containers[1].HostAddr}
	c.Assert(hosts[0], check.Equals, net.URLToHost(node1.URL()))
	c.Assert(hosts[1], check.Equals, net.URLToHost(node1.URL()))

	c.Assert(eventtest.EventDesc{
		Target: event.Target{Type: "container", Value: toMoveCont.ID},
		Kind:   "healer",
		StartCustomData: map[string]interface{}{
			"hostaddr": "127.0.0.1",
			"id":       toMoveCont.ID,
		},
		ErrorMatches: `.*Error trying to heal containers.*`,
		EndCustomData: map[string]interface{}{
			"hostaddr": "",
		},
	}, eventtest.HasEvent)
}
Example #10
0
func (s *S) TestRunContainerHealerAlreadyHealed(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	node1 := p.Servers()[0]
	app := provisiontest.NewFakeApp("myapp", "python", 0)
	_, err = p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)

	containers := p.AllContainers()
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
	c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)

	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	node1.PrepareFailure("createError", "/containers/create")

	healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
	healer.healContainerIfNeeded(toMoveCont)
	healer.healContainerIfNeeded(toMoveCont)

	expected := dockertest.ContainerMoving{
		ContainerID: toMoveCont.ID,
		HostFrom:    toMoveCont.HostAddr,
		HostTo:      "",
	}
	movings := p.Movings()
	c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})

	healingColl, err := healingCollection()
	c.Assert(err, check.IsNil)
	defer healingColl.Close()
	var events []HealingEvent
	err = healingColl.Find(nil).All(&events)
	c.Assert(err, check.IsNil)
	c.Assert(events, check.HasLen, 1)
	c.Assert(events[0].Action, check.Equals, "container-healing")
	c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
	c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
	c.Assert(events[0].Error, check.Equals, "")
	c.Assert(events[0].Successful, check.Equals, true)
	c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
	c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
Example #11
0
func (s *S) TestRunContainerHealerShutdown(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()

	node1 := p.Servers()[0]
	app := provisiontest.NewFakeApp("myapp", "python", 0)
	_, err = p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)

	containers := p.AllContainers()
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
	c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})

	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)

	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	node1.PrepareFailure("createError", "/containers/create")

	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Done:                make(chan bool),
		Locker:              dockertest.NewFakeLocker(),
	})
	ch := make(chan bool)
	go func() {
		defer close(ch)
		healer.RunContainerHealer()
	}()
	healer.Shutdown()
	<-ch

	expected := dockertest.ContainerMoving{
		ContainerID: toMoveCont.ID,
		HostFrom:    toMoveCont.HostAddr,
		HostTo:      "",
	}
	movings := p.Movings()
	c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})
}
Example #12
0
func (s *S) TestRunContainerHealerCreatedContainer(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	app := provisiontest.NewFakeApp("myapp", "python", 2)
	node1 := p.Servers()[0]
	containers, err := p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
	toMoveCont := containers[1]
	toMoveCont.MongoID = bson.NewObjectIdWithTime(time.Now().Add(-2 * time.Minute))
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	node1.PrepareFailure("createError", "/containers/create")
	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()
	expected := []dockertest.ContainerMoving{
		{
			ContainerID: toMoveCont.ID,
			HostFrom:    toMoveCont.HostAddr,
			HostTo:      "",
		},
	}
	movings := p.Movings()
	c.Assert(movings, check.DeepEquals, expected)
	healingColl, err := healingCollection()
	c.Assert(err, check.IsNil)
	defer healingColl.Close()
	var events []HealingEvent
	err = healingColl.Find(nil).All(&events)
	c.Assert(err, check.IsNil)
	c.Assert(events, check.HasLen, 1)
	c.Assert(events[0].Action, check.Equals, "container-healing")
	c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
	c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
	c.Assert(events[0].Error, check.Equals, "")
	c.Assert(events[0].Successful, check.Equals, true)
	c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
	c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
Example #13
0
func (s *S) TestRunContainerHealerAlreadyHealed(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	node1 := p.Servers()[0]
	app := newFakeAppInDB("myapp", "python", 0)
	_, err = p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	containers := p.AllContainers()
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
	c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	node1.PrepareFailure("createError", "/containers/create")
	healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
	err = healer.healContainerIfNeeded(toMoveCont)
	c.Assert(err, check.IsNil)
	err = healer.healContainerIfNeeded(toMoveCont)
	c.Assert(err, check.IsNil)
	expected := dockertest.ContainerMoving{
		ContainerID: toMoveCont.ID,
		HostFrom:    toMoveCont.HostAddr,
		HostTo:      "",
	}
	movings := p.Movings()
	c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})
	c.Assert(eventtest.EventDesc{
		Target: event.Target{Type: "container", Value: toMoveCont.ID},
		Kind:   "healer",
		StartCustomData: map[string]interface{}{
			"hostaddr": "127.0.0.1",
			"id":       toMoveCont.ID,
		},
		EndCustomData: map[string]interface{}{
			"hostaddr": "127.0.0.1",
			"id":       bson.M{"$ne": ""},
		},
	}, eventtest.HasEvent)
}
Example #14
0
func (s *S) TestRunContainerHealerStoppedContainer(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	app := newFakeAppInDB("myapp", "python", 2)
	node1 := p.Servers()[0]
	containers, err := p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Dead: true})
	toMoveCont := containers[1]
	err = toMoveCont.SetStatus(p, provision.StatusStopped, false)
	c.Assert(err, check.IsNil)
	toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	node1.PrepareFailure("createError", "/containers/create")
	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()
	expected := []dockertest.ContainerMoving{
		{
			ContainerID: toMoveCont.ID,
			HostFrom:    toMoveCont.HostAddr,
			HostTo:      "",
		},
	}
	movings := p.Movings()
	c.Assert(movings, check.DeepEquals, expected)
	c.Assert(eventtest.EventDesc{
		Target: event.Target{Type: "container", Value: toMoveCont.ID},
		Kind:   "healer",
		StartCustomData: map[string]interface{}{
			"hostaddr": "127.0.0.1",
			"id":       toMoveCont.ID,
		},
		EndCustomData: map[string]interface{}{
			"hostaddr": "127.0.0.1",
			"id":       bson.M{"$ne": ""},
		},
	}, eventtest.HasEvent)
}
Example #15
0
func (s *S) TestListUnresponsiveContainersNoHostPort(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	var result []container.Container
	coll := p.Collection()
	defer coll.Close()
	now := time.Now().UTC()
	coll.Insert(
		container.Container{ID: "c1", AppName: "app_time_test", LastSuccessStatusUpdate: now.Add(-10 * time.Minute)},
	)
	defer coll.RemoveAll(bson.M{"appname": "app_time_test"})
	result, err = listUnresponsiveContainers(p, 3*time.Minute)
	c.Assert(err, check.IsNil)
	c.Assert(result, check.HasLen, 0)
}
Example #16
0
func (s *S) TestRemoveNamedContainers(c *check.C) {
	config.Set("docker:bs:image", "myregistry/tsuru/bs")
	_, err := nodecontainer.InitializeBS()
	c.Assert(err, check.IsNil)
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	nodes, err := p.Cluster().Nodes()
	c.Assert(err, check.IsNil)
	for i, n := range nodes {
		n.Metadata["pool"] = fmt.Sprintf("p-%d", i)
		_, err = p.Cluster().UpdateNode(n)
		c.Assert(err, check.IsNil)
	}
	server := p.Servers()[0]
	var paths []string
	server.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		paths = append(paths, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery)
		server.DefaultHandler().ServeHTTP(w, r)
	}))
	server2 := p.Servers()[1]
	var paths2 []string
	server2.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		paths2 = append(paths2, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery)
		server2.DefaultHandler().ServeHTTP(w, r)
	}))
	err = ensureContainersStarted(p, ioutil.Discard, true, nil)
	c.Assert(err, check.IsNil)
	paths = nil
	paths2 = nil
	expectedPaths := []string{"POST /containers/big-sibling/stop?t=10", "DELETE /containers/big-sibling?force=1"}
	err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "")
	c.Assert(err, check.IsNil)
	c.Assert(paths, check.DeepEquals, expectedPaths)
	c.Assert(paths2, check.DeepEquals, expectedPaths)
	err = ensureContainersStarted(p, ioutil.Discard, true, nil)
	c.Assert(err, check.IsNil)
	paths = nil
	paths2 = nil
	err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "p-1")
	c.Assert(err, check.IsNil)
	c.Assert(paths, check.DeepEquals, []string(nil))
	c.Assert(paths2, check.DeepEquals, expectedPaths)
}
Example #17
0
func (s *S) TestListUnresponsiveContainers(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	var result []container.Container
	coll := p.Collection()
	defer coll.Close()
	now := time.Now().UTC()
	coll.Insert(
		container.Container{ID: "c1", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now, HostPort: "80"},
		container.Container{ID: "c2", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now.Add(-1 * time.Minute), HostPort: "80"},
		container.Container{ID: "c3", AppName: "app_time_test", ProcessName: "p", LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80"},
	)
	defer coll.RemoveAll(bson.M{"appname": "app_time_test"})
	result, err = listUnresponsiveContainers(p, 3*time.Minute)
	c.Assert(err, check.IsNil)
	c.Assert(len(result), check.Equals, 1)
	c.Assert(result[0].ID, check.Equals, "c3")
}
Example #18
0
func (s *S) TestRecreateBsContainers(c *check.C) {
	_, err := nodecontainer.InitializeBS()
	c.Assert(err, check.IsNil)
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	var buf safe.Buffer
	err = recreateContainers(p, &buf)
	c.Assert(err, check.IsNil)
	nodes, err := p.Cluster().Nodes()
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 2)
	client, err := nodes[0].Client()
	c.Assert(err, check.IsNil)
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err := client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
	client, err = nodes[1].Client()
	c.Assert(err, check.IsNil)
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err = client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
	// It runs in parallel, so we check both ordering
	output1 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s []
relaunching node container "big-sibling" in the node %s []
`, nodes[0].Address, nodes[1].Address)
	output2 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s []
relaunching node container "big-sibling" in the node %s []
`, nodes[1].Address, nodes[0].Address)
	if got := buf.String(); got != output1 && got != output2 {
		c.Errorf("Wrong output:\n%s", got)
	}
}
Example #19
0
func (s *S) TestRunContainerHealerThrottled(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	node1 := p.Servers()[0]
	app := newFakeAppInDB("myapp", "python", 0)
	_, err = p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)

	containers := p.AllContainers()
	c.Assert(containers, check.HasLen, 2)
	c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
	for i := 0; i < 3; i++ {
		var evt *event.Event
		evt, err = event.NewInternal(&event.Opts{
			Target:       event.Target{Type: "container", Value: toMoveCont.ID},
			InternalKind: "healer",
			CustomData:   toMoveCont,
			Allowed:      event.Allowed(permission.PermAppReadEvents),
		})
		c.Assert(err, check.IsNil)
		err = evt.DoneCustomData(nil, nil)
		c.Assert(err, check.IsNil)
	}
	healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
	err = healer.healContainerIfNeeded(toMoveCont)
	c.Assert(err, check.ErrorMatches, "Error trying to insert container healing event, healing aborted: event throttled, limit for healer on container \".*?\" is 3 every 5m0s")
}
Example #20
0
func (s *S) TestListUnresponsiveContainersIncludeStopped(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	var result []container.Container
	coll := p.Collection()
	defer coll.Close()
	now := time.Now().UTC()
	coll.Insert(
		container.Container{ID: "c1", AppName: "app_time_test",
			LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80", Status: provision.StatusStopped.String()},
		container.Container{ID: "c2", AppName: "app_time_test",
			LastSuccessStatusUpdate: now.Add(-5 * time.Minute), HostPort: "80", Status: provision.StatusStarted.String()},
	)
	defer coll.RemoveAll(bson.M{"appname": "app_time_test"})
	result, err = listUnresponsiveContainers(p, 3*time.Minute)
	c.Assert(err, check.IsNil)
	c.Assert(result, check.HasLen, 2)
	ids := []string{result[0].ID, result[1].ID}
	sort.Strings(ids)
	c.Assert(ids, check.DeepEquals, []string{"c1", "c2"})
}
Example #21
0
func (s *S) TestClusterHookBeforeCreateContainerStartsStopped(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	var buf safe.Buffer
	err = RecreateContainers(p, &buf)
	c.Assert(err, check.IsNil)
	nodes, err := p.Cluster().Nodes()
	c.Assert(err, check.IsNil)
	client, err := nodes[0].Client()
	c.Assert(err, check.IsNil)
	err = client.StopContainer("big-sibling", 1)
	c.Assert(err, check.IsNil)
	contData, err := client.InspectContainer("big-sibling")
	c.Assert(err, check.IsNil)
	c.Assert(contData.State.Running, check.Equals, false)
	hook := ClusterHook{Provisioner: p}
	err = hook.BeforeCreateContainer(nodes[0])
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 2)
	client, err = nodes[0].Client()
	c.Assert(err, check.IsNil)
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err := client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, "big-sibling")
	c.Assert(container.State.Running, check.Equals, true)
	client, err = nodes[1].Client()
	c.Assert(err, check.IsNil)
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 1)
	container, err = client.InspectContainer(containers[0].ID)
	c.Assert(err, check.IsNil)
	c.Assert(container.Name, check.Equals, "big-sibling")
	c.Assert(container.State.Running, check.Equals, true)
}
Example #22
0
func (s *S) TestRunContainerHealerStoppedContainerAlreadyStopped(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	app := newFakeAppInDB("myapp", "python", 2)
	node1 := p.Servers()[0]
	containers, err := p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
	toMoveCont := containers[1]
	err = toMoveCont.SetStatus(p, provision.StatusStopped, false)
	c.Assert(err, check.IsNil)
	toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	node1.PrepareFailure("createError", "/containers/create")
	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()
	movings := p.Movings()
	c.Assert(movings, check.IsNil)
	c.Assert(eventtest.EventDesc{
		IsEmpty: true,
	}, eventtest.HasEvent)
	conts, err := p.ListContainers(bson.M{"id": toMoveCont.ID})
	c.Assert(err, check.IsNil)
	c.Assert(conts, check.HasLen, 1)
	c.Assert(conts[0].Status, check.Equals, provision.StatusStopped.String())
	c.Assert(time.Since(conts[0].LastSuccessStatusUpdate) < time.Minute, check.Equals, true)
}
Example #23
0
func (s *S) TestHealContainer(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	containers := []container.Container{
		{ID: "cont1", AppName: "app1"},
		{ID: "cont2", AppName: "app1"},
		{ID: "cont3", AppName: "app2"},
	}
	p.SetContainers("localhost", containers)
	locker := dockertest.NewFakeLocker()
	locked := locker.Lock(containers[0].AppName)
	c.Assert(locked, check.Equals, true)
	defer locker.Unlock(containers[0].AppName)
	healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: locker})
	_, err = healer.healContainer(containers[0])
	c.Assert(err, check.IsNil)
	expected := []container.Container{
		{ID: "cont1-recreated", HostAddr: "localhost", AppName: "app1"},
		{ID: "cont2", AppName: "app1", HostAddr: "localhost"},
		{ID: "cont3", AppName: "app2", HostAddr: "localhost"},
	}
	c.Assert(p.Containers("localhost"), check.DeepEquals, expected)
}
Example #24
0
func (s *S) TestRunContainerHealer(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	app := provisiontest.NewFakeApp("myapp", "python", 2)
	node1 := p.Servers()[0]
	containers, err := p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})

	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)

	node1.PrepareFailure("createError", "/containers/create")

	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()

	expected := []dockertest.ContainerMoving{
		{
			ContainerID: toMoveCont.ID,
			HostFrom:    toMoveCont.HostAddr,
			HostTo:      "",
		},
	}
	movings := p.Movings()
	c.Assert(movings, check.DeepEquals, expected)
	queries := p.Queries()
	c.Assert(queries, check.HasLen, 1)
	queryTime := queries[0]["lastsuccessstatusupdate"].(bson.M)["$lt"].(time.Time)
	delete(queries[0], "lastsuccessstatusupdate")
	c.Assert(time.Now().UTC().Add(-1*time.Minute).Sub(queryTime) < time.Second, check.Equals, true)
	c.Assert(queries, check.DeepEquals, []bson.M{{
		"$or": []bson.M{
			{"hostport": bson.M{"$ne": ""}},
			{"processname": bson.M{"$ne": ""}},
		},
		"status": bson.M{"$nin": []string{
			provision.StatusStopped.String(),
			provision.StatusBuilding.String(),
			provision.StatusAsleep.String(),
		}},
	}})
	healingColl, err := healingCollection()
	c.Assert(err, check.IsNil)
	defer healingColl.Close()
	var events []HealingEvent
	err = healingColl.Find(nil).All(&events)
	c.Assert(err, check.IsNil)
	c.Assert(events, check.HasLen, 1)
	c.Assert(events[0].Action, check.Equals, "container-healing")
	c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
	c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
	c.Assert(events[0].Error, check.Equals, "")
	c.Assert(events[0].Successful, check.Equals, true)
	c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
	c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
Example #25
0
func (s *S) TestRunContainerHealerWithError(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	node1 := p.Servers()[0]
	app := provisiontest.NewFakeApp("myapp", "python", 0)
	_, err = p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)

	containers := p.AllContainers()
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
	c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})

	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
	p.FailMove(
		errors.New("cannot move container"),
		errors.New("cannot move container"),
		errors.New("cannot move container"),
	)

	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()

	containers = p.AllContainers()
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
	hosts := []string{containers[0].HostAddr, containers[1].HostAddr}
	c.Assert(hosts[0], check.Equals, net.URLToHost(node1.URL()))
	c.Assert(hosts[1], check.Equals, net.URLToHost(node1.URL()))

	healingColl, err := healingCollection()
	c.Assert(err, check.IsNil)
	defer healingColl.Close()
	var events []HealingEvent
	err = healingColl.Find(nil).All(&events)
	c.Assert(err, check.IsNil)
	c.Assert(events, check.HasLen, 1)
	c.Assert(events[0].Action, check.Equals, "container-healing")
	c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
	c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
	c.Assert(events[0].Error, check.Matches, "(?s).*Error trying to heal containers.*")
	c.Assert(events[0].Successful, check.Equals, false)
	c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
	c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "")
}
Example #26
0
func (s *S) TestRunContainerHealer(c *check.C) {
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	app := newFakeAppInDB("myapp", "python", 2)
	node1 := p.Servers()[0]
	containers, err := p.StartContainers(dockertest.StartContainersArgs{
		Endpoint:  node1.URL(),
		App:       app,
		Amount:    map[string]int{"web": 2},
		Image:     "tsuru/python",
		PullImage: true,
	})
	c.Assert(err, check.IsNil)
	node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
	node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})

	toMoveCont := containers[1]
	toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
	p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)

	node1.PrepareFailure("createError", "/containers/create")

	healer := NewContainerHealer(ContainerHealerArgs{
		Provisioner:         p,
		MaxUnresponsiveTime: time.Minute,
		Locker:              dockertest.NewFakeLocker(),
	})
	healer.runContainerHealerOnce()

	expected := []dockertest.ContainerMoving{
		{
			ContainerID: toMoveCont.ID,
			HostFrom:    toMoveCont.HostAddr,
			HostTo:      "",
		},
	}
	movings := p.Movings()
	c.Assert(movings, check.DeepEquals, expected)
	queries := p.Queries()
	c.Assert(queries, check.HasLen, 1)
	queryTime := queries[0]["lastsuccessstatusupdate"].(bson.M)["$lt"].(time.Time)
	delete(queries[0], "lastsuccessstatusupdate")
	c.Assert(time.Now().UTC().Add(-1*time.Minute).Sub(queryTime) < time.Second, check.Equals, true)
	c.Assert(queries, check.DeepEquals, []bson.M{{
		"id":      bson.M{"$ne": ""},
		"appname": bson.M{"$ne": ""},
		"$or": []bson.M{
			{"hostport": bson.M{"$ne": ""}},
			{"processname": bson.M{"$ne": ""}},
		},
		"status": bson.M{"$nin": []string{
			provision.StatusBuilding.String(),
			provision.StatusAsleep.String(),
		}},
	}})
	c.Assert(eventtest.EventDesc{
		Target: event.Target{Type: "container", Value: toMoveCont.ID},
		Kind:   "healer",
		StartCustomData: map[string]interface{}{
			"hostaddr": "127.0.0.1",
			"id":       toMoveCont.ID,
		},
		EndCustomData: map[string]interface{}{
			"hostaddr": "127.0.0.1",
			"id":       bson.M{"$ne": ""},
		},
	}, eventtest.HasEvent)
}
Example #27
0
func (s *S) TestEnsureContainersStarted(c *check.C) {
	c1 := NodeContainerConfig{
		Name: "bs",
		Config: docker.Config{
			Image: "bsimg",
			Env: []string{
				"A=1",
				"B=2",
			},
		},
		HostConfig: docker.HostConfig{
			RestartPolicy: docker.AlwaysRestart(),
			Privileged:    true,
			Binds:         []string{"/xyz:/abc:rw"},
		},
	}
	err := AddNewContainer("", &c1)
	c.Assert(err, check.IsNil)
	c2 := c1
	c2.Name = "sysdig"
	c2.Config.Image = "sysdigimg"
	c2.Config.Env = []string{"X=Z"}
	err = AddNewContainer("", &c2)
	c.Assert(err, check.IsNil)
	p, err := dockertest.StartMultipleServersCluster()
	c.Assert(err, check.IsNil)
	var createBodies []string
	var names []string
	var mut sync.Mutex
	server := p.Servers()[0]
	server.CustomHandler("/containers/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mut.Lock()
		defer mut.Unlock()
		data, _ := ioutil.ReadAll(r.Body)
		createBodies = append(createBodies, string(data))
		names = append(names, r.URL.Query().Get("name"))
		r.Body = ioutil.NopCloser(bytes.NewBuffer(data))
		server.DefaultHandler().ServeHTTP(w, r)
	}))
	defer p.Destroy()
	buf := safe.NewBuffer(nil)
	err = ensureContainersStarted(p, buf, true)
	c.Assert(err, check.IsNil)
	parts := strings.Split(buf.String(), "\n")
	c.Assert(parts, check.HasLen, 5)
	sort.Strings(parts)
	c.Assert(parts[1], check.Matches, `relaunching node container "bs" in the node http://127.0.0.1:\d+/ \[\]`)
	c.Assert(parts[2], check.Matches, `relaunching node container "bs" in the node http://localhost:\d+/ \[\]`)
	c.Assert(parts[3], check.Matches, `relaunching node container "sysdig" in the node http://127.0.0.1:\d+/ \[\]`)
	c.Assert(parts[4], check.Matches, `relaunching node container "sysdig" in the node http://localhost:\d+/ \[\]`)
	c.Assert(createBodies, check.HasLen, 2)
	c.Assert(names, check.HasLen, 2)
	sort.Strings(names)
	c.Assert(names, check.DeepEquals, []string{"bs", "sysdig"})
	sort.Strings(createBodies)
	result := make([]struct {
		docker.Config
		HostConfig docker.HostConfig
	}, 2)
	err = json.Unmarshal([]byte(createBodies[0]), &result[0])
	c.Assert(err, check.IsNil)
	err = json.Unmarshal([]byte(createBodies[1]), &result[1])
	c.Assert(err, check.IsNil)
	c.Assert(result, check.DeepEquals, []struct {
		docker.Config
		HostConfig docker.HostConfig
	}{
		{
			Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "A=1", "B=2"}, Image: "bsimg"},
			HostConfig: docker.HostConfig{
				Binds:         []string{"/xyz:/abc:rw"},
				Privileged:    true,
				RestartPolicy: docker.RestartPolicy{Name: "always"},
				LogConfig:     docker.LogConfig{},
			},
		},
		{
			Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "X=Z"}, Image: "sysdigimg"},
			HostConfig: docker.HostConfig{
				Binds:         []string{"/xyz:/abc:rw"},
				Privileged:    true,
				RestartPolicy: docker.RestartPolicy{Name: "always"},
				LogConfig:     docker.LogConfig{},
			},
		},
	})
	conf := configFor("bs")
	var result1 NodeContainerConfig
	err = conf.Load("", &result1)
	c.Assert(err, check.IsNil)
	c.Assert(result1.PinnedImage, check.Equals, "")
	client, err := docker.NewClient(p.Servers()[0].URL())
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
	client, err = docker.NewClient(p.Servers()[1].URL())
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(containers, check.HasLen, 2)
}