func (s *S) TestHealerHandleErrorThrottled(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
healer.started = time.Now().Add(-3 * time.Second)
conf := healerConfig()
err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
for i := 0; i < 3; i++ {
var evt *event.Event
evt, err = event.NewInternal(&event.Opts{
Target: event.Target{Type: "node", Value: nodes[0].Address()},
InternalKind: "healer",
Allowed: event.Allowed(permission.PermPoolReadEvents),
})
c.Assert(err, check.IsNil)
err = evt.Done(nil)
c.Assert(err, check.IsNil)
}
err = healer.tryHealingNode(nodes[0], "myreason", nil)
c.Assert(err, check.ErrorMatches, "Error trying to insert node healing event, healing aborted: event throttled, limit for healer on node \".*?\" is 3 every 5m0s")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
}