// RecreateContainers relaunch all bs containers in the cluster for the given
// DockerProvisioner, logging progress to the given writer.
//
// It assumes that the given writer is thread safe.
func RecreateContainers(p DockerProvisioner, w io.Writer) error {
cluster := p.Cluster()
nodes, err := cluster.UnfilteredNodes()
if err != nil {
return err
}
errChan := make(chan error, len(nodes))
wg := sync.WaitGroup{}
log.Debugf("[bs containers] recreating %d containers", len(nodes))
for i := range nodes {
wg.Add(1)
go func(i int) {
defer wg.Done()
node := &nodes[i]
pool := node.Metadata["pool"]
log.Debugf("[bs containers] recreating container in %s [%s]", node.Address, pool)
fmt.Fprintf(w, "relaunching bs container in the node %s [%s]\n", node.Address, pool)
err := createContainer(node.Address, pool, p, true)
if err != nil {
msg := fmt.Sprintf("[bs containers] failed to create container in %s [%s]: %s", node.Address, pool, err)
log.Error(msg)
err = errors.New(msg)
errChan <- err
}
}(i)
}
wg.Wait()
close(errChan)
return <-errChan
}
func (s *S) TestHealerHealNode(c *check.C) {
rollback := startTestRepositoryServer()
defer rollback()
defer func() {
machines, _ := iaas.ListMachines()
for _, m := range machines {
m.Destroy()
}
}()
factory, iaasInst := newHealerIaaSConstructorWithInst("127.0.0.1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.addr = "localhost"
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", urlPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
cluster, err := cluster.New(nil, &cluster.MapStorage{},
cluster.Node{Address: node1.URL()},
)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
var p dockerProvisioner
err = p.Initialize()
c.Assert(err, check.IsNil)
p.cluster = cluster
defer p.Destroy(appInstance)
p.Provision(appInstance)
imageId, err := appCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
customData := map[string]interface{}{
"procfile": "web: python ./myapp",
}
err = saveImageCustomData(imageId, customData)
c.Assert(err, check.IsNil)
_, err = addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "127.0.0.1",
toAdd: map[string]*containersToAdd{"web": {Quantity: 1}},
app: appInstance,
imageId: imageId,
provisioner: &p,
})
c.Assert(err, check.IsNil)
conn, err := db.Conn()
c.Assert(err, check.IsNil)
defer conn.Close()
appStruct := &app.App{
Name: appInstance.GetName(),
}
err = conn.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
defer conn.Apps().Remove(bson.M{"name": appStruct.Name})
healer := nodeHealer{
locks: make(map[string]*sync.Mutex),
provisioner: &p,
disabledTime: 0,
failuresBeforeHealing: 1,
waitTimeNewMachine: 1 * time.Second,
}
nodes, err := p.getCluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1")
containers, err := p.listAllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
c.Assert(containers[0].HostAddr, check.Equals, "127.0.0.1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "127.0.0.1")
nodes[0].Metadata["iaas"] = "my-healer-iaas"
created, err := healer.healNode(&nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, fmt.Sprintf("http://localhost:%d", urlPort(node2.URL())))
nodes, err = cluster.UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node2.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "localhost")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "localhost")
done := make(chan bool)
go func() {
for range time.Tick(100 * time.Millisecond) {
containers, err := p.listAllContainers()
if err == nil && len(containers) == 1 && containers[0].HostAddr == "localhost" {
close(done)
return
}
}
}()
select {
case <-done:
case <-time.After(5 * time.Second):
c.Fatal("Timed out waiting for containers to move")
}
}
func (s *S) TestHealerHandleError(c *check.C) {
rollback := startTestRepositoryServer()
defer rollback()
defer func() {
machines, _ := iaas.ListMachines()
for _, m := range machines {
m.Destroy()
}
}()
factory, iaasInst := newHealerIaaSConstructorWithInst("127.0.0.1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.addr = "localhost"
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", urlPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
cluster, err := cluster.New(nil, &cluster.MapStorage{},
cluster.Node{Address: node1.URL()},
)
c.Assert(err, check.IsNil)
var p dockerProvisioner
err = p.Initialize()
c.Assert(err, check.IsNil)
p.cluster = cluster
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
imageId, err := appCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
customData := map[string]interface{}{
"procfile": "web: python ./myapp",
}
err = saveImageCustomData(imageId, customData)
c.Assert(err, check.IsNil)
_, err = addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "127.0.0.1",
toAdd: map[string]*containersToAdd{"web": {Quantity: 1}},
app: appInstance,
imageId: imageId,
provisioner: &p,
})
c.Assert(err, check.IsNil)
conn, err := db.Conn()
c.Assert(err, check.IsNil)
defer conn.Close()
appStruct := &app.App{
Name: appInstance.GetName(),
}
err = conn.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
defer conn.Apps().Remove(bson.M{"name": appStruct.Name})
healer := nodeHealer{
locks: make(map[string]*sync.Mutex),
provisioner: &p,
disabledTime: 0,
failuresBeforeHealing: 1,
waitTimeNewMachine: 1 * time.Second,
}
nodes, err := cluster.UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "127.0.0.1")
nodes[0].Metadata["iaas"] = "my-healer-iaas"
nodes[0].Metadata["Failures"] = "2"
waitTime := healer.HandleError(&nodes[0])
c.Assert(waitTime, check.Equals, time.Duration(0))
nodes, err = cluster.UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node2.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "localhost")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "localhost")
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []healingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "node-healing")
c.Assert(events[0].StartTime, check.Not(check.DeepEquals), time.Time{})
c.Assert(events[0].EndTime, check.Not(check.DeepEquals), time.Time{})
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingNode.Address, check.Equals, fmt.Sprintf("http://127.0.0.1:%d/", urlPort(node1.URL())))
c.Assert(events[0].CreatedNode.Address, check.Equals, fmt.Sprintf("http://localhost:%d", urlPort(node2.URL())))
}