Beispiel #1
0
func (s *S) TestHealerHandleErrorThrottled(c *check.C) {
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")
	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)
	node, err := p.GetNode("http://addr1:1")
	c.Assert(err, check.IsNil)
	healer := newNodeHealer(nodeHealerArgs{
		FailuresBeforeHealing: 1,
		WaitTimeNewMachine:    time.Minute,
	})
	healer.Shutdown()
	healer.started = time.Now().Add(-3 * time.Second)
	conf := healerConfig()
	err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
	c.Assert(err, check.IsNil)
	err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
	c.Assert(err, check.IsNil)
	time.Sleep(1200 * time.Millisecond)
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")
	for i := 0; i < 3; i++ {
		var evt *event.Event
		evt, err = event.NewInternal(&event.Opts{
			Target:       event.Target{Type: "node", Value: nodes[0].Address()},
			InternalKind: "healer",
			Allowed:      event.Allowed(permission.PermPoolReadEvents),
		})
		c.Assert(err, check.IsNil)
		err = evt.Done(nil)
		c.Assert(err, check.IsNil)
	}
	err = healer.tryHealingNode(nodes[0], "myreason", nil)
	c.Assert(err, check.ErrorMatches, "Error trying to insert node healing event, healing aborted: event throttled, limit for healer on node \".*?\" is 3 every 5m0s")
	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
}