func (s *S) TestHealerHealNodeDestroyError(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaasInst.DelErr = fmt.Errorf("my destroy error")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
buf := bytes.Buffer{}
log.SetLogger(log.NewWriterLogger(&buf, false))
defer log.SetLogger(nil)
created, err := healer.healNode(nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, "http://addr2:2")
c.Assert(buf.String(), check.Matches, "(?s).*my destroy error.*")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
}
func machinesList(w http.ResponseWriter, r *http.Request, token auth.Token) error {
machines, err := iaas.ListMachines()
if err != nil {
return err
}
contexts := permission.ContextsForPermission(token, permission.PermMachineRead)
allowedIaaS := map[string]struct{}{}
for _, c := range contexts {
if c.CtxType == permission.CtxGlobal {
allowedIaaS = nil
break
}
if c.CtxType == permission.CtxIaaS {
allowedIaaS[c.Value] = struct{}{}
}
}
for i := 0; allowedIaaS != nil && i < len(machines); i++ {
if _, ok := allowedIaaS[machines[i].Iaas]; !ok {
machines = append(machines[:i], machines[i+1:]...)
i--
}
}
err = json.NewEncoder(w).Encode(machines)
if err != nil {
return err
}
return nil
}
func (s *S) TestHealerHealNode(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
created, err := healer.healNode(nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, "http://addr2:2")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
}
func (s *S) TestHealerHandleErrorThrottled(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
healer.started = time.Now().Add(-3 * time.Second)
conf := healerConfig()
err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
for i := 0; i < 3; i++ {
var evt *event.Event
evt, err = event.NewInternal(&event.Opts{
Target: event.Target{Type: "node", Value: nodes[0].Address()},
InternalKind: "healer",
Allowed: event.Allowed(permission.PermPoolReadEvents),
})
c.Assert(err, check.IsNil)
err = evt.Done(nil)
c.Assert(err, check.IsNil)
}
err = healer.tryHealingNode(nodes[0], "myreason", nil)
c.Assert(err, check.ErrorMatches, "Error trying to insert node healing event, healing aborted: event throttled, limit for healer on node \".*?\" is 3 every 5m0s")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
}
func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) {
defer func() {
machines, _ := iaas.ListMachines()
for _, m := range machines {
m.Destroy()
}
}()
iaas.RegisterIaasProvider("my-healer-iaas", newHealerIaaSConstructor("127.0.0.1", nil))
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaas.RegisterIaasProvider("my-healer-iaas", newHealerIaaSConstructor("localhost", nil))
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2.PrepareFailure("ping-failure", "/_ping")
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", urlPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
cluster, err := cluster.New(nil, &cluster.MapStorage{},
cluster.Node{Address: node1.URL()},
)
c.Assert(err, check.IsNil)
node1.PrepareFailure("pingErr", "/_ping")
cluster.StartActiveMonitoring(100 * time.Millisecond)
time.Sleep(300 * time.Millisecond)
cluster.StopActiveMonitoring()
var p dockerProvisioner
err = p.Initialize()
c.Assert(err, check.IsNil)
p.cluster = cluster
healer := nodeHealer{
locks: make(map[string]*sync.Mutex),
provisioner: &p,
disabledTime: 0,
failuresBeforeHealing: 1,
waitTimeNewMachine: 1 * time.Second,
}
nodes, err := p.getCluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1")
c.Assert(nodes[0].FailureCount() > 0, check.Equals, true)
nodes[0].Metadata["iaas"] = "my-healer-iaas"
created, err := healer.healNode(&nodes[0])
c.Assert(err, check.ErrorMatches, ".*error registering new node.*")
c.Assert(created.Address, check.Equals, "")
c.Assert(nodes[0].FailureCount(), check.Equals, 0)
nodes, err = p.getCluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1")
}
func machinesList(w http.ResponseWriter, r *http.Request, token auth.Token) error {
machines, err := iaas.ListMachines()
if err != nil {
return err
}
err = json.NewEncoder(w).Encode(machines)
if err != nil {
return err
}
return nil
}
func (s *S) SetUpTest(c *check.C) {
config.Set("docker:api-timeout", 2)
conn, err := db.Conn()
c.Assert(err, check.IsNil)
defer conn.Close()
dbtest.ClearAllCollections(conn.Apps().Database)
queue.ResetQueue()
iaas.ResetAll()
machines, _ := iaas.ListMachines()
for _, m := range machines {
m.Destroy()
}
os.Setenv("TSURU_TARGET", "http://localhost")
}
//listNodeHandler call scheduler.Nodes to list all nodes into it.
func listNodeHandler(w http.ResponseWriter, r *http.Request, t auth.Token) error {
nodeList, err := mainDockerProvisioner.Cluster().UnfilteredNodes()
if err != nil {
return err
}
machines, err := iaas.ListMachines()
if err != nil {
return err
}
result := map[string]interface{}{
"nodes": nodeList,
"machines": machines,
}
return json.NewEncoder(w).Encode(result)
}
func listNodesHandler(w http.ResponseWriter, r *http.Request, t auth.Token) error {
pools, err := listContextValues(t, permission.PermNodeRead, false)
if err != nil {
return err
}
nodes, err := mainDockerProvisioner.Cluster().UnfilteredNodes()
if err != nil {
return err
}
if pools != nil {
filteredNodes := make([]cluster.Node, 0, len(nodes))
for _, node := range nodes {
for _, pool := range pools {
if node.Metadata["pool"] == pool {
filteredNodes = append(filteredNodes, node)
break
}
}
}
nodes = filteredNodes
}
iaases, err := listContextValues(t, permission.PermMachineRead, false)
if err != nil {
return err
}
machines, err := iaas.ListMachines()
if err != nil {
return err
}
if iaases != nil {
filteredMachines := make([]iaas.Machine, 0, len(machines))
for _, machine := range machines {
for _, iaas := range iaases {
if machine.Iaas == iaas {
filteredMachines = append(filteredMachines, machine)
break
}
}
}
machines = filteredMachines
}
result := map[string]interface{}{
"nodes": nodes,
"machines": machines,
}
return json.NewEncoder(w).Encode(result)
}
func (s *S) TestTryHealingNodeDoubleCheck(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.started = time.Now().Add(-3 * time.Second)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
healErr := healer.tryHealingNode(nodes[0], "something", nil)
c.Assert(healErr, check.IsNil)
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
c.Assert(eventtest.EventDesc{
IsEmpty: true,
}, eventtest.HasEvent)
}
// title: list nodes
// path: /{provisioner}/node
// method: GET
// produce: application/json
// responses:
// 200: Ok
// 204: No content
func listNodesHandler(w http.ResponseWriter, r *http.Request, t auth.Token) error {
pools, err := permission.ListContextValues(t, permission.PermNodeRead, false)
if err != nil {
return err
}
provs, err := provision.Registry()
if err != nil {
return err
}
provNameMap := map[string]string{}
var allNodes []provision.Node
for _, prov := range provs {
nodeProv, ok := prov.(provision.NodeProvisioner)
if !ok {
continue
}
var nodes []provision.Node
nodes, err = nodeProv.ListNodes(nil)
if err != nil {
return err
}
for _, n := range nodes {
provNameMap[n.Address()] = prov.GetName()
}
allNodes = append(allNodes, nodes...)
}
if pools != nil {
filteredNodes := make([]provision.Node, 0, len(allNodes))
for _, node := range allNodes {
for _, pool := range pools {
if node.Pool() == pool {
filteredNodes = append(filteredNodes, node)
break
}
}
}
allNodes = filteredNodes
}
iaases, err := permission.ListContextValues(t, permission.PermMachineRead, false)
if err != nil {
return err
}
machines, err := iaas.ListMachines()
if err != nil {
return err
}
if iaases != nil {
filteredMachines := make([]iaas.Machine, 0, len(machines))
for _, machine := range machines {
for _, iaas := range iaases {
if machine.Iaas == iaas {
filteredMachines = append(filteredMachines, machine)
break
}
}
}
machines = filteredMachines
}
if len(allNodes) == 0 && len(machines) == 0 {
w.WriteHeader(http.StatusNoContent)
return nil
}
nodesJson := make([]json.RawMessage, len(allNodes))
for i, n := range allNodes {
nodesJson[i], err = provision.NodeToJSON(n)
if err != nil {
return err
}
}
result := listNodeResponse{
Nodes: nodesJson,
Machines: machines,
}
w.Header().Set("Content-Type", "application/json")
return json.NewEncoder(w).Encode(result)
}
func (s *S) TestHealerHandleError(c *check.C) {
factory, iaasInst := dockertest.NewHealerIaaSConstructorWithInst("127.0.0.1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "localhost"
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", dockertest.URLPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p, err := s.newFakeDockerProvisioner(node1.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
app := provisiontest.NewFakeApp("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 1},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
healer := NewNodeHealer(NodeHealerArgs{
Provisioner: p,
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
nodes, err := p.Cluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "127.0.0.1")
nodes[0].Metadata["iaas"] = "my-healer-iaas"
nodes[0].Metadata["Failures"] = "2"
waitTime := healer.HandleError(&nodes[0])
c.Assert(waitTime, check.Equals, time.Duration(0))
nodes, err = p.Cluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node2.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "localhost")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "localhost")
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "node-healing")
c.Assert(events[0].StartTime, check.Not(check.DeepEquals), time.Time{})
c.Assert(events[0].EndTime, check.Not(check.DeepEquals), time.Time{})
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingNode.Address, check.Equals, fmt.Sprintf("http://127.0.0.1:%d/", dockertest.URLPort(node1.URL())))
c.Assert(events[0].CreatedNode.Address, check.Equals, fmt.Sprintf("http://localhost:%d", dockertest.URLPort(node2.URL())))
}
func (s *S) TestHealerHealNode(c *check.C) {
factory, iaasInst := dockertest.NewHealerIaaSConstructorWithInst("127.0.0.1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "localhost"
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", dockertest.URLPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p, err := s.newFakeDockerProvisioner(node1.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
app := provisiontest.NewFakeApp("myapp", "python", 0)
_, err = p.StartContainers(dockertest.StartContainersArgs{
Endpoint: node1.URL(),
App: app,
Amount: map[string]int{"web": 1},
Image: "tsuru/python",
PullImage: true,
})
c.Assert(err, check.IsNil)
healer := NewNodeHealer(NodeHealerArgs{
Provisioner: p,
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
nodes, err := p.Cluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1")
containers := p.AllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
c.Assert(containers[0].HostAddr, check.Equals, "127.0.0.1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "127.0.0.1")
nodes[0].Metadata["iaas"] = "my-healer-iaas"
created, err := healer.healNode(&nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, fmt.Sprintf("http://localhost:%d", dockertest.URLPort(node2.URL())))
nodes, err = p.Cluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node2.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "localhost")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "localhost")
err = tsurutest.WaitCondition(5*time.Second, func() bool {
containers := p.AllContainers()
return len(containers) == 1 && containers[0].HostAddr == "localhost"
})
c.Assert(err, check.IsNil)
}
func (s *S) TestHealerHealNode(c *check.C) {
rollback := startTestRepositoryServer()
defer rollback()
defer func() {
machines, _ := iaas.ListMachines()
for _, m := range machines {
m.Destroy()
}
}()
factory, iaasInst := newHealerIaaSConstructorWithInst("127.0.0.1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.addr = "localhost"
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", urlPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
cluster, err := cluster.New(nil, &cluster.MapStorage{},
cluster.Node{Address: node1.URL()},
)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
var p dockerProvisioner
err = p.Initialize()
c.Assert(err, check.IsNil)
p.cluster = cluster
defer p.Destroy(appInstance)
p.Provision(appInstance)
imageId, err := appCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
customData := map[string]interface{}{
"procfile": "web: python ./myapp",
}
err = saveImageCustomData(imageId, customData)
c.Assert(err, check.IsNil)
_, err = addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "127.0.0.1",
toAdd: map[string]*containersToAdd{"web": {Quantity: 1}},
app: appInstance,
imageId: imageId,
provisioner: &p,
})
c.Assert(err, check.IsNil)
conn, err := db.Conn()
c.Assert(err, check.IsNil)
defer conn.Close()
appStruct := &app.App{
Name: appInstance.GetName(),
}
err = conn.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
defer conn.Apps().Remove(bson.M{"name": appStruct.Name})
healer := nodeHealer{
locks: make(map[string]*sync.Mutex),
provisioner: &p,
disabledTime: 0,
failuresBeforeHealing: 1,
waitTimeNewMachine: 1 * time.Second,
}
nodes, err := p.getCluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1")
containers, err := p.listAllContainers()
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
c.Assert(containers[0].HostAddr, check.Equals, "127.0.0.1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "127.0.0.1")
nodes[0].Metadata["iaas"] = "my-healer-iaas"
created, err := healer.healNode(&nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, fmt.Sprintf("http://localhost:%d", urlPort(node2.URL())))
nodes, err = cluster.UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node2.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "localhost")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "localhost")
done := make(chan bool)
go func() {
for range time.Tick(100 * time.Millisecond) {
containers, err := p.listAllContainers()
if err == nil && len(containers) == 1 && containers[0].HostAddr == "localhost" {
close(done)
return
}
}
}()
select {
case <-done:
case <-time.After(5 * time.Second):
c.Fatal("Timed out waiting for containers to move")
}
}
func (s *S) TestHealerHandleError(c *check.C) {
rollback := startTestRepositoryServer()
defer rollback()
defer func() {
machines, _ := iaas.ListMachines()
for _, m := range machines {
m.Destroy()
}
}()
factory, iaasInst := newHealerIaaSConstructorWithInst("127.0.0.1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.addr = "localhost"
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", urlPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
cluster, err := cluster.New(nil, &cluster.MapStorage{},
cluster.Node{Address: node1.URL()},
)
c.Assert(err, check.IsNil)
var p dockerProvisioner
err = p.Initialize()
c.Assert(err, check.IsNil)
p.cluster = cluster
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
imageId, err := appCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
customData := map[string]interface{}{
"procfile": "web: python ./myapp",
}
err = saveImageCustomData(imageId, customData)
c.Assert(err, check.IsNil)
_, err = addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "127.0.0.1",
toAdd: map[string]*containersToAdd{"web": {Quantity: 1}},
app: appInstance,
imageId: imageId,
provisioner: &p,
})
c.Assert(err, check.IsNil)
conn, err := db.Conn()
c.Assert(err, check.IsNil)
defer conn.Close()
appStruct := &app.App{
Name: appInstance.GetName(),
}
err = conn.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
defer conn.Apps().Remove(bson.M{"name": appStruct.Name})
healer := nodeHealer{
locks: make(map[string]*sync.Mutex),
provisioner: &p,
disabledTime: 0,
failuresBeforeHealing: 1,
waitTimeNewMachine: 1 * time.Second,
}
nodes, err := cluster.UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "127.0.0.1")
nodes[0].Metadata["iaas"] = "my-healer-iaas"
nodes[0].Metadata["Failures"] = "2"
waitTime := healer.HandleError(&nodes[0])
c.Assert(waitTime, check.Equals, time.Duration(0))
nodes, err = cluster.UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node2.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "localhost")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "localhost")
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []healingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 1)
c.Assert(events[0].Action, check.Equals, "node-healing")
c.Assert(events[0].StartTime, check.Not(check.DeepEquals), time.Time{})
c.Assert(events[0].EndTime, check.Not(check.DeepEquals), time.Time{})
c.Assert(events[0].Error, check.Equals, "")
c.Assert(events[0].Successful, check.Equals, true)
c.Assert(events[0].FailingNode.Address, check.Equals, fmt.Sprintf("http://127.0.0.1:%d/", urlPort(node1.URL())))
c.Assert(events[0].CreatedNode.Address, check.Equals, fmt.Sprintf("http://localhost:%d", urlPort(node2.URL())))
}
func (s *S) TestHealerHandleErrorFailureEvent(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
healer.started = time.Now().Add(-3 * time.Second)
conf := healerConfig()
err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
p.PrepareFailure("AddNode", fmt.Errorf("error registering new node"))
nodes[0].(*provisiontest.FakeNode).SetHealth(2, true)
waitTime := healer.HandleError(nodes[0].(provision.NodeHealthChecker))
c.Assert(waitTime, check.Equals, time.Duration(0))
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "node", Value: "http://addr1:1"},
Kind: "healer",
StartCustomData: map[string]interface{}{
"reason": "2 consecutive failures",
"node._id": "http://addr1:1",
},
ErrorMatches: `Can't auto-heal after 2 failures for node addr1: error registering new node: error registering new node`,
}, eventtest.HasEvent)
}
func (s *S) TestCheckActiveHealing(c *check.C) {
conf := healerConfig()
err := conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err = iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
healer.started = time.Now().Add(-3 * time.Second)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
healer.runActiveHealing()
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "node", Value: "http://addr1:1"},
Kind: "healer",
StartCustomData: map[string]interface{}{
"reason": bson.M{"$regex": `last update \d+\.\d*?s ago, last success \d+\.\d*?s ago`},
"lastcheck.time": bson.M{"$exists": true},
"node._id": "http://addr1:1",
},
EndCustomData: map[string]interface{}{
"_id": "http://addr2:2",
},
}, eventtest.HasEvent)
}
func (s *S) TestTryHealingNodeConcurrent(c *check.C) {
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(10))
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.started = time.Now().Add(-3 * time.Second)
conf := healerConfig()
err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
wg := sync.WaitGroup{}
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
defer wg.Done()
healErr := healer.tryHealingNode(nodes[0], "something", nil)
c.Assert(healErr, check.IsNil)
}()
}
wg.Wait()
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: "node", Value: "http://addr1:1"},
Kind: "healer",
StartCustomData: map[string]interface{}{
"reason": "something",
"node._id": "http://addr1:1",
},
EndCustomData: map[string]interface{}{
"_id": "http://addr2:2",
},
}, eventtest.HasEvent)
}