func (s *S) TestRunContainerHealerMaxCounterExceeded(c *check.C) {
conts := []container.Container{
{ID: "cont1"}, {ID: "cont2"}, {ID: "cont3"}, {ID: "cont4"},
{ID: "cont5"}, {ID: "cont6"}, {ID: "cont7"}, {ID: "cont8"},
}
for i := 0; i < len(conts)-1; i++ {
evt, err := NewHealingEvent(conts[i])
c.Assert(err, check.IsNil)
err = evt.Update(conts[i+1], nil)
c.Assert(err, check.IsNil)
}
toMoveCont := conts[7]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
p, err := dockertest.NewFakeDockerProvisioner()
c.Assert(err, check.IsNil)
defer p.Destroy()
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.ErrorMatches, "Containers healing: number of healings for container cont8 in the last 30 minutes exceeds limit of 3: 7")
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 7)
}
func (s *S) TestRunContainerHealerRemovedFromDB(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := newFakeAppInDB("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 1},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
p.DeleteContainer(containers[0].ID)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[0]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.IsNil)
}
func (s *S) TestRunContainerHealerDoesntHealWhenContainerIsRestarting(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := provisiontest.NewFakeApp("myapp", "python", 0)
cont, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 1},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(cont[0].ID, docker.State{Running: false, Restarting: true})
toMoveCont := cont[0]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
p.PrepareListResult([]container.Container{toMoveCont}, nil)
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 0)
}
func (s *S) TestRunContainerHealerCreatedContainerNoProcess(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := provisiontest.NewFakeApp("myapp", "python", 2)
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: p.Servers()[0].URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
notToMove := containers[1]
notToMove.MongoID = bson.NewObjectIdWithTime(time.Now().Add(-2 * time.Minute))
notToMove.ProcessName = ""
p.PrepareListResult([]container.Container{containers[0], notToMove}, nil)
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
movings := p.Movings()
c.Assert(movings, check.IsNil)
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
n, err := healingColl.Count()
c.Assert(err, check.IsNil)
c.Assert(n, check.Equals, 0)
}
func (s *S) TestRunContainerHealerWithError(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := newFakeAppInDB("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
p.FailMove(
errors.New("cannot move container"),
errors.New("cannot move container"),
errors.New("cannot move container"),
)
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
containers = p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
hosts := []string{containers[0].HostAddr, containers[1].HostAddr}
c.Assert(hosts[0], check.Equals, net.URLToHost(node1.URL()))
c.Assert(hosts[1], check.Equals, net.URLToHost(node1.URL()))
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
Kind: "healer",
StartCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": toMoveCont.ID,
},
ErrorMatches: `.*Error trying to heal containers.*`,
EndCustomData: map[string]interface{}{
"hostaddr": "",
},
}, eventtest.HasEvent)
}
func (s *S) TestRunContainerHealerAlreadyHealed(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := provisiontest.NewFakeApp("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
healer.healContainerIfNeeded(toMoveCont)
healer.healContainerIfNeeded(toMoveCont)
expected := dockertest.ContainerMoving{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "container-healing")
c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
func (s *S) TestRunContainerHealerShutdown(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := provisiontest.NewFakeApp("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Done: make(chan bool),
Locker: dockertest.NewFakeLocker(),
})
ch := make(chan bool)
go func() {
defer close(ch)
healer.RunContainerHealer()
}()
healer.Shutdown()
<-ch
expected := dockertest.ContainerMoving{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})
}
func (s *S) TestRunContainerHealerCreatedContainer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := provisiontest.NewFakeApp("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.MongoID = bson.NewObjectIdWithTime(time.Now().Add(-2 * time.Minute))
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
expected := []dockertest.ContainerMoving{
{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
},
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, expected)
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "container-healing")
c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
func (s *S) TestRunContainerHealerAlreadyHealed(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := newFakeAppInDB("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.IsNil)
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.IsNil)
expected := dockertest.ContainerMoving{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, []dockertest.ContainerMoving{expected})
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
Kind: "healer",
StartCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": toMoveCont.ID,
},
EndCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": bson.M{"$ne": ""},
},
}, eventtest.HasEvent)
}
func (s *S) TestRunContainerHealerStoppedContainer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := newFakeAppInDB("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Dead: true})
toMoveCont := containers[1]
err = toMoveCont.SetStatus(p, provision.StatusStopped, false)
c.Assert(err, check.IsNil)
toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
expected := []dockertest.ContainerMoving{
{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
},
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, expected)
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
Kind: "healer",
StartCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": toMoveCont.ID,
},
EndCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": bson.M{"$ne": ""},
},
}, eventtest.HasEvent)
}
func (s *S) TestRunContainerHealerThrottled(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := newFakeAppInDB("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-5 * time.Minute)
for i := 0; i < 3; i++ {
var evt *event.Event
evt, err = event.NewInternal(&event.Opts{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
InternalKind: "healer",
CustomData: toMoveCont,
Allowed: event.Allowed(permission.PermAppReadEvents),
})
c.Assert(err, check.IsNil)
err = evt.DoneCustomData(nil, nil)
c.Assert(err, check.IsNil)
}
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.ErrorMatches, "Error trying to insert container healing event, healing aborted: event throttled, limit for healer on container \".*?\" is 3 every 5m0s")
}
func (s *S) TestRunContainerHealerStoppedContainerAlreadyStopped(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := newFakeAppInDB("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
err = toMoveCont.SetStatus(p, provision.StatusStopped, false)
c.Assert(err, check.IsNil)
toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
movings := p.Movings()
c.Assert(movings, check.IsNil)
c.Assert(eventtest.EventDesc{
IsEmpty: true,
}, eventtest.HasEvent)
conts, err := p.ListContainers(bson.M{"id": toMoveCont.ID})
c.Assert(err, check.IsNil)
c.Assert(conts, check.HasLen, 1)
c.Assert(conts[0].Status, check.Equals, provision.StatusStopped.String())
c.Assert(time.Since(conts[0].LastSuccessStatusUpdate) < time.Minute, check.Equals, true)
}
func (s *S) TestHealContainer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
containers := []container.Container{
{ID: "cont1", AppName: "app1"},
{ID: "cont2", AppName: "app1"},
{ID: "cont3", AppName: "app2"},
}
p.SetContainers("localhost", containers)
locker := dockertest.NewFakeLocker()
locked := locker.Lock(containers[0].AppName)
c.Assert(locked, check.Equals, true)
defer locker.Unlock(containers[0].AppName)
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: locker})
_, err = healer.healContainer(containers[0])
c.Assert(err, check.IsNil)
expected := []container.Container{
{ID: "cont1-recreated", HostAddr: "localhost", AppName: "app1"},
{ID: "cont2", AppName: "app1", HostAddr: "localhost"},
{ID: "cont3", AppName: "app2", HostAddr: "localhost"},
}
c.Assert(p.Containers("localhost"), check.DeepEquals, expected)
}
func (s *S) TestRunContainerHealer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := provisiontest.NewFakeApp("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
expected := []dockertest.ContainerMoving{
{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
},
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, expected)
queries := p.Queries()
c.Assert(queries, check.HasLen, 1)
queryTime := queries[0]["lastsuccessstatusupdate"].(bson.M)["$lt"].(time.Time)
delete(queries[0], "lastsuccessstatusupdate")
c.Assert(time.Now().UTC().Add(-1*time.Minute).Sub(queryTime) < time.Second, check.Equals, true)
c.Assert(queries, check.DeepEquals, []bson.M{{
"$or": []bson.M{
{"hostport": bson.M{"$ne": ""}},
{"processname": bson.M{"$ne": ""}},
},
"status": bson.M{"$nin": []string{
provision.StatusStopped.String(),
provision.StatusBuilding.String(),
provision.StatusAsleep.String(),
}},
}})
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "container-healing")
c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "127.0.0.1")
}
func (s *S) TestRunContainerHealerWithError(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
node1 := p.Servers()[0]
app := provisiontest.NewFakeApp("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].HostAddr, check.Equals, net.URLToHost(node1.URL()))
c.Assert(containers[1].HostAddr, check.Equals, net.URLToHost(node1.URL()))
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
p.FailMove(
errors.New("cannot move container"),
errors.New("cannot move container"),
errors.New("cannot move container"),
)
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
containers = p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
hosts := []string{containers[0].HostAddr, containers[1].HostAddr}
c.Assert(hosts[0], check.Equals, net.URLToHost(node1.URL()))
c.Assert(hosts[1], check.Equals, net.URLToHost(node1.URL()))
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "container-healing")
c.Assert(events[0].StartTime.IsZero(), check.Equals, false)
c.Assert(events[0].EndTime.IsZero(), check.Equals, false)
c.Assert(events[0].Error, check.Matches, "(?s).*Error trying to heal containers.*")
c.Assert(events[0].Successful, check.Equals, false)
c.Assert(events[0].FailingContainer.HostAddr, check.Equals, "127.0.0.1")
c.Assert(events[0].CreatedContainer.HostAddr, check.Equals, "")
}
func (s *S) TestRunContainerHealer(c *check.C) {
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
app := newFakeAppInDB("myapp", "python", 2)
node1 := p.Servers()[0]
containers, err := p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 2},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
node1.MutateContainer(containers[0].ID, docker.State{Running: false, Restarting: false})
node1.MutateContainer(containers[1].ID, docker.State{Running: false, Restarting: false})
toMoveCont := containers[1]
toMoveCont.LastSuccessStatusUpdate = time.Now().UTC().Add(-5 * time.Minute)
p.PrepareListResult([]container.Container{containers[0], toMoveCont}, nil)
node1.PrepareFailure("createError", "/containers/create")
healer := NewContainerHealer(ContainerHealerArgs{
Provisioner: p,
MaxUnresponsiveTime: time.Minute,
Locker: dockertest.NewFakeLocker(),
})
healer.runContainerHealerOnce()
expected := []dockertest.ContainerMoving{
{
ContainerID: toMoveCont.ID,
HostFrom: toMoveCont.HostAddr,
HostTo: "",
},
}
movings := p.Movings()
c.Assert(movings, check.DeepEquals, expected)
queries := p.Queries()
c.Assert(queries, check.HasLen, 1)
queryTime := queries[0]["lastsuccessstatusupdate"].(bson.M)["$lt"].(time.Time)
delete(queries[0], "lastsuccessstatusupdate")
c.Assert(time.Now().UTC().Add(-1*time.Minute).Sub(queryTime) < time.Second, check.Equals, true)
c.Assert(queries, check.DeepEquals, []bson.M{{
"id": bson.M{"$ne": ""},
"appname": bson.M{"$ne": ""},
"$or": []bson.M{
{"hostport": bson.M{"$ne": ""}},
{"processname": bson.M{"$ne": ""}},
},
"status": bson.M{"$nin": []string{
provision.StatusBuilding.String(),
provision.StatusAsleep.String(),
}},
}})
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "container", Value: toMoveCont.ID},
Kind: "healer",
StartCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": toMoveCont.ID,
},
EndCustomData: map[string]interface{}{
"hostaddr": "127.0.0.1",
"id": bson.M{"$ne": ""},
},
}, eventtest.HasEvent)
}