Esempio n. 1
0
func (s *S) TestHealerHandleErrorThrottled(c *check.C) {
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")
	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)
	node, err := p.GetNode("http://addr1:1")
	c.Assert(err, check.IsNil)
	healer := newNodeHealer(nodeHealerArgs{
		FailuresBeforeHealing: 1,
		WaitTimeNewMachine:    time.Minute,
	})
	healer.Shutdown()
	healer.started = time.Now().Add(-3 * time.Second)
	conf := healerConfig()
	err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
	c.Assert(err, check.IsNil)
	err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
	c.Assert(err, check.IsNil)
	time.Sleep(1200 * time.Millisecond)
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")
	for i := 0; i < 3; i++ {
		var evt *event.Event
		evt, err = event.NewInternal(&event.Opts{
			Target:       event.Target{Type: "node", Value: nodes[0].Address()},
			InternalKind: "healer",
			Allowed:      event.Allowed(permission.PermPoolReadEvents),
		})
		c.Assert(err, check.IsNil)
		err = evt.Done(nil)
		c.Assert(err, check.IsNil)
	}
	err = healer.tryHealingNode(nodes[0], "myreason", nil)
	c.Assert(err, check.ErrorMatches, "Error trying to insert node healing event, healing aborted: event throttled, limit for healer on node \".*?\" is 3 every 5m0s")
	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
}
Esempio n. 2
0
func (s *S) TestHealerHealNodeDestroyError(c *check.C) {
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaasInst.DelErr = fmt.Errorf("my destroy error")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")
	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)

	healer := newNodeHealer(nodeHealerArgs{
		WaitTimeNewMachine: time.Minute,
	})
	healer.Shutdown()
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")

	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")

	buf := bytes.Buffer{}
	log.SetLogger(log.NewWriterLogger(&buf, false))
	defer log.SetLogger(nil)
	created, err := healer.healNode(nodes[0])
	c.Assert(err, check.IsNil)
	c.Assert(created.Address, check.Equals, "http://addr2:2")
	c.Assert(buf.String(), check.Matches, "(?s).*my destroy error.*")

	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")

	machines, err = iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr2")
}
Esempio n. 3
0
func (s *S) TestHealerHealNode(c *check.C) {
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")
	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)

	healer := newNodeHealer(nodeHealerArgs{
		FailuresBeforeHealing: 1,
		WaitTimeNewMachine:    time.Minute,
	})
	healer.Shutdown()
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")

	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")

	created, err := healer.healNode(nodes[0])
	c.Assert(err, check.IsNil)
	c.Assert(created.Address, check.Equals, "http://addr2:2")
	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")

	machines, err = iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr2")
}
Esempio n. 4
0
func (s *S) TestTryHealingNodeDoubleCheck(c *check.C) {
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")
	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)
	healer := newNodeHealer(nodeHealerArgs{
		FailuresBeforeHealing: 1,
		WaitTimeNewMachine:    time.Minute,
	})
	healer.started = time.Now().Add(-3 * time.Second)
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	healErr := healer.tryHealingNode(nodes[0], "something", nil)
	c.Assert(healErr, check.IsNil)
	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")
	c.Assert(eventtest.EventDesc{
		IsEmpty: true,
	}, eventtest.HasEvent)
}
Esempio n. 5
0
func (s *S) TestHealerHealNodeCreateMachineError(c *check.C) {
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	iaasInst.Err = fmt.Errorf("my create machine error")

	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)

	healer := newNodeHealer(nodeHealerArgs{
		FailuresBeforeHealing: 1,
		WaitTimeNewMachine:    time.Minute,
	})
	healer.Shutdown()
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
	fakeNode := nodes[0].(*provisiontest.FakeNode)
	fakeNode.SetHealth(1, false)
	c.Assert(fakeNode.FailureCount() > 0, check.Equals, true)
	created, err := healer.healNode(nodes[0])
	c.Assert(err, check.ErrorMatches, ".*my create machine error.*")
	c.Assert(created, check.IsNil)
	c.Assert(fakeNode.FailureCount(), check.Equals, 0)
	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
	c.Assert(nodes[0].Status(), check.Equals, "enabled")
}
Esempio n. 6
0
func (s *S) TestTryHealingNodeConcurrent(c *check.C) {
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(10))
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")
	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)
	node, err := p.GetNode("http://addr1:1")
	c.Assert(err, check.IsNil)
	healer := newNodeHealer(nodeHealerArgs{
		FailuresBeforeHealing: 1,
		WaitTimeNewMachine:    time.Minute,
	})
	healer.started = time.Now().Add(-3 * time.Second)
	conf := healerConfig()
	err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
	c.Assert(err, check.IsNil)
	err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
	c.Assert(err, check.IsNil)
	time.Sleep(1200 * time.Millisecond)
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")
	wg := sync.WaitGroup{}
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			healErr := healer.tryHealingNode(nodes[0], "something", nil)
			c.Assert(healErr, check.IsNil)
		}()
	}
	wg.Wait()
	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
	machines, err = iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr2")
	c.Assert(eventtest.EventDesc{
		Target: event.Target{Type: "node", Value: "http://addr1:1"},
		Kind:   "healer",
		StartCustomData: map[string]interface{}{
			"reason":   "something",
			"node._id": "http://addr1:1",
		},
		EndCustomData: map[string]interface{}{
			"_id": "http://addr2:2",
		},
	}, eventtest.HasEvent)
}
Esempio n. 7
0
func (s *S) TestCheckActiveHealing(c *check.C) {
	conf := healerConfig()
	err := conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
	c.Assert(err, check.IsNil)
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err = iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")

	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)
	node, err := p.GetNode("http://addr1:1")
	c.Assert(err, check.IsNil)

	healer := newNodeHealer(nodeHealerArgs{
		WaitTimeNewMachine: time.Minute,
	})
	healer.Shutdown()
	healer.started = time.Now().Add(-3 * time.Second)

	err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
	c.Assert(err, check.IsNil)
	time.Sleep(1200 * time.Millisecond)

	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")

	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")

	healer.runActiveHealing()

	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")

	machines, err = iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr2")

	c.Assert(eventtest.EventDesc{
		Target: event.Target{Type: "node", Value: "http://addr1:1"},
		Kind:   "healer",
		StartCustomData: map[string]interface{}{
			"reason":         bson.M{"$regex": `last update \d+\.\d*?s ago, last success \d+\.\d*?s ago`},
			"lastcheck.time": bson.M{"$exists": true},
			"node._id":       "http://addr1:1",
		},
		EndCustomData: map[string]interface{}{
			"_id": "http://addr2:2",
		},
	}, eventtest.HasEvent)
}
Esempio n. 8
0
func (s *S) TestHealerHandleErrorFailureEvent(c *check.C) {
	factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
	iaas.RegisterIaasProvider("my-healer-iaas", factory)
	_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
	c.Assert(err, check.IsNil)
	iaasInst.Addr = "addr2"
	config.Set("iaas:node-protocol", "http")
	config.Set("iaas:node-port", 2)
	defer config.Unset("iaas:node-protocol")
	defer config.Unset("iaas:node-port")
	p := provisiontest.ProvisionerInstance
	err = p.AddNode(provision.AddNodeOptions{
		Address:  "http://addr1:1",
		Metadata: map[string]string{"iaas": "my-healer-iaas"},
	})
	c.Assert(err, check.IsNil)
	node, err := p.GetNode("http://addr1:1")
	c.Assert(err, check.IsNil)

	healer := newNodeHealer(nodeHealerArgs{
		FailuresBeforeHealing: 1,
		WaitTimeNewMachine:    time.Minute,
	})
	healer.Shutdown()
	healer.started = time.Now().Add(-3 * time.Second)
	conf := healerConfig()
	err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
	c.Assert(err, check.IsNil)
	err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
	c.Assert(err, check.IsNil)
	time.Sleep(1200 * time.Millisecond)
	nodes, err := p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")

	machines, err := iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")

	p.PrepareFailure("AddNode", fmt.Errorf("error registering new node"))
	nodes[0].(*provisiontest.FakeNode).SetHealth(2, true)

	waitTime := healer.HandleError(nodes[0].(provision.NodeHealthChecker))
	c.Assert(waitTime, check.Equals, time.Duration(0))

	nodes, err = p.ListNodes(nil)
	c.Assert(err, check.IsNil)
	c.Assert(nodes, check.HasLen, 1)
	c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")

	machines, err = iaas.ListMachines()
	c.Assert(err, check.IsNil)
	c.Assert(machines, check.HasLen, 1)
	c.Assert(machines[0].Address, check.Equals, "addr1")

	c.Assert(eventtest.EventDesc{
		Target: event.Target{Type: "node", Value: "http://addr1:1"},
		Kind:   "healer",
		StartCustomData: map[string]interface{}{
			"reason":   "2 consecutive failures",
			"node._id": "http://addr1:1",
		},
		ErrorMatches: `Can't auto-heal after 2 failures for node addr1: error registering new node: error registering new node`,
	}, eventtest.HasEvent)
}