func (s *S) TestHealerHandleErrorThrottled(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
healer.started = time.Now().Add(-3 * time.Second)
conf := healerConfig()
err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
for i := 0; i < 3; i++ {
var evt *event.Event
evt, err = event.NewInternal(&event.Opts{
Target: event.Target{Type: "node", Value: nodes[0].Address()},
InternalKind: "healer",
Allowed: event.Allowed(permission.PermPoolReadEvents),
})
c.Assert(err, check.IsNil)
err = evt.Done(nil)
c.Assert(err, check.IsNil)
}
err = healer.tryHealingNode(nodes[0], "myreason", nil)
c.Assert(err, check.ErrorMatches, "Error trying to insert node healing event, healing aborted: event throttled, limit for healer on node \".*?\" is 3 every 5m0s")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
}
func (s *S) TestHealerHealNodeDestroyError(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaasInst.DelErr = fmt.Errorf("my destroy error")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
buf := bytes.Buffer{}
log.SetLogger(log.NewWriterLogger(&buf, false))
defer log.SetLogger(nil)
created, err := healer.healNode(nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, "http://addr2:2")
c.Assert(buf.String(), check.Matches, "(?s).*my destroy error.*")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
}
func (s *S) TestHealerHealNode(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
created, err := healer.healNode(nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, "http://addr2:2")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
}
func (s *S) TestTryHealingNodeDoubleCheck(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.started = time.Now().Add(-3 * time.Second)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
healErr := healer.tryHealingNode(nodes[0], "something", nil)
c.Assert(healErr, check.IsNil)
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
c.Assert(eventtest.EventDesc{
IsEmpty: true,
}, eventtest.HasEvent)
}
func (s *S) TestHealerHealNodeCreateMachineError(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
iaasInst.Err = fmt.Errorf("my create machine error")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
fakeNode := nodes[0].(*provisiontest.FakeNode)
fakeNode.SetHealth(1, false)
c.Assert(fakeNode.FailureCount() > 0, check.Equals, true)
created, err := healer.healNode(nodes[0])
c.Assert(err, check.ErrorMatches, ".*my create machine error.*")
c.Assert(created, check.IsNil)
c.Assert(fakeNode.FailureCount(), check.Equals, 0)
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
c.Assert(nodes[0].Status(), check.Equals, "enabled")
}
func (s *S) TestTryHealingNodeConcurrent(c *check.C) {
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(10))
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.started = time.Now().Add(-3 * time.Second)
conf := healerConfig()
err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
wg := sync.WaitGroup{}
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
defer wg.Done()
healErr := healer.tryHealingNode(nodes[0], "something", nil)
c.Assert(healErr, check.IsNil)
}()
}
wg.Wait()
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "node", Value: "http://addr1:1"},
Kind: "healer",
StartCustomData: map[string]interface{}{
"reason": "something",
"node._id": "http://addr1:1",
},
EndCustomData: map[string]interface{}{
"_id": "http://addr2:2",
},
}, eventtest.HasEvent)
}
func (s *S) TestCheckActiveHealing(c *check.C) {
conf := healerConfig()
err := conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err = iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
healer.started = time.Now().Add(-3 * time.Second)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
healer.runActiveHealing()
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "node", Value: "http://addr1:1"},
Kind: "healer",
StartCustomData: map[string]interface{}{
"reason": bson.M{"$regex": `last update \d+\.\d*?s ago, last success \d+\.\d*?s ago`},
"lastcheck.time": bson.M{"$exists": true},
"node._id": "http://addr1:1",
},
EndCustomData: map[string]interface{}{
"_id": "http://addr2:2",
},
}, eventtest.HasEvent)
}
func (s *S) TestHealerHandleErrorFailureEvent(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
healer.started = time.Now().Add(-3 * time.Second)
conf := healerConfig()
err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
p.PrepareFailure("AddNode", fmt.Errorf("error registering new node"))
nodes[0].(*provisiontest.FakeNode).SetHealth(2, true)
waitTime := healer.HandleError(nodes[0].(provision.NodeHealthChecker))
c.Assert(waitTime, check.Equals, time.Duration(0))
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "node", Value: "http://addr1:1"},
Kind: "healer",
StartCustomData: map[string]interface{}{
"reason": "2 consecutive failures",
"node._id": "http://addr1:1",
},
ErrorMatches: `Can't auto-heal after 2 failures for node addr1: error registering new node: error registering new node`,
}, eventtest.HasEvent)
}