func (s *S) TestHealerHandleErrorDoesntTriggerEventIfHealingCountTooLarge(c *check.C) { nodes := []cluster.Node{ {Address: "addr1"}, {Address: "addr2"}, {Address: "addr3"}, {Address: "addr4"}, {Address: "addr5"}, {Address: "addr6"}, {Address: "addr7"}, {Address: "addr8"}, } for i := 0; i < len(nodes)-1; i++ { evt, err := NewHealingEvent(nodes[i]) c.Assert(err, check.IsNil) err = evt.Update(nodes[i+1], nil) c.Assert(err, check.IsNil) } iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("127.0.0.1", nil)) healer := NewNodeHealer(NodeHealerArgs{ DisabledTime: 20, FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Minute, }) nodes[7].Metadata = map[string]string{ "Failures": "2", "LastSuccess": "something", "iaas": "my-healer-iaas", } waitTime := healer.HandleError(&nodes[7]) c.Assert(waitTime, check.Equals, time.Duration(20)) healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() var events []HealingEvent err = healingColl.Find(nil).All(&events) c.Assert(err, check.IsNil) c.Assert(events, check.HasLen, 7) }
func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) { iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("127.0.0.1", nil)) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("localhost", nil)) node1, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) node2, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) node2.PrepareFailure("ping-failure", "/_ping") config.Set("iaas:node-protocol", "http") config.Set("iaas:node-port", dockertest.URLPort(node2.URL())) defer config.Unset("iaas:node-protocol") defer config.Unset("iaas:node-port") p, err := s.newFakeDockerProvisioner(node1.URL()) c.Assert(err, check.IsNil) defer p.Destroy() node1.PrepareFailure("pingErr", "/_ping") p.Cluster().StartActiveMonitoring(100 * time.Millisecond) time.Sleep(300 * time.Millisecond) p.Cluster().StopActiveMonitoring() healer := NewNodeHealer(NodeHealerArgs{ Provisioner: p, FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Second, }) nodes, err := p.Cluster().UnfilteredNodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL())) c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1") c.Assert(nodes[0].FailureCount() > 0, check.Equals, true) nodes[0].Metadata["iaas"] = "my-healer-iaas" created, err := healer.healNode(&nodes[0]) c.Assert(err, check.ErrorMatches, ".*timeout waiting for result.*") c.Assert(created.Address, check.Equals, "") c.Assert(nodes[0].FailureCount(), check.Equals, 0) nodes, err = p.Cluster().Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL())) c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1") }