func (s *S) TestHealerHandleErrorDoesntTriggerEventIfHealingCountTooLarge(c *check.C) {
nodes := []cluster.Node{
{Address: "addr1"}, {Address: "addr2"}, {Address: "addr3"}, {Address: "addr4"},
{Address: "addr5"}, {Address: "addr6"}, {Address: "addr7"}, {Address: "addr8"},
}
for i := 0; i < len(nodes)-1; i++ {
evt, err := NewHealingEvent(nodes[i])
c.Assert(err, check.IsNil)
err = evt.Update(nodes[i+1], nil)
c.Assert(err, check.IsNil)
}
iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("127.0.0.1", nil))
healer := NewNodeHealer(NodeHealerArgs{
DisabledTime: 20,
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
nodes[7].Metadata = map[string]string{
"Failures": "2",
"LastSuccess": "something",
"iaas": "my-healer-iaas",
}
waitTime := healer.HandleError(&nodes[7])
c.Assert(waitTime, check.Equals, time.Duration(20))
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 7)
}
func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) {
iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("127.0.0.1", nil))
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("localhost", nil))
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2.PrepareFailure("ping-failure", "/_ping")
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", dockertest.URLPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p, err := s.newFakeDockerProvisioner(node1.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
node1.PrepareFailure("pingErr", "/_ping")
p.Cluster().StartActiveMonitoring(100 * time.Millisecond)
time.Sleep(300 * time.Millisecond)
p.Cluster().StopActiveMonitoring()
healer := NewNodeHealer(NodeHealerArgs{
Provisioner: p,
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Second,
})
nodes, err := p.Cluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1")
c.Assert(nodes[0].FailureCount() > 0, check.Equals, true)
nodes[0].Metadata["iaas"] = "my-healer-iaas"
created, err := healer.healNode(&nodes[0])
c.Assert(err, check.ErrorMatches, ".*timeout waiting for result.*")
c.Assert(created.Address, check.Equals, "")
c.Assert(nodes[0].FailureCount(), check.Equals, 0)
nodes, err = p.Cluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1")
}
