func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) {
iaas.RegisterIaasProvider("my-healer-iaas", iaasTesting.NewHealerIaaSConstructor("addr1", nil))
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaas.RegisterIaasProvider("my-healer-iaas", iaasTesting.NewHealerIaaSConstructor("addr2", nil))
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
p.PrepareFailure("AddNode", fmt.Errorf("add node error"))
healer := newNodeHealer(nodeHealerArgs{
WaitTimeNewMachine: time.Second,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
created, err := healer.healNode(nodes[0])
c.Assert(err, check.ErrorMatches, ".*error registering new node: add node error.*")
c.Assert(created, check.IsNil)
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
c.Assert(nodes[0].Status(), check.Equals, "enabled")
}
func (s *HandlersSuite) TestAddNodeHandlerCreatingAnIaasMachineExplicit(c *check.C) {
server, waitQueue := s.startFakeDockerNode(c)
defer server.Stop()
iaas.RegisterIaasProvider("test-iaas", newTestIaaS)
iaas.RegisterIaasProvider("another-test-iaas", newTestIaaS)
mainDockerProvisioner.cluster, _ = cluster.New(&segregatedScheduler{}, &cluster.MapStorage{})
opts := provision.AddPoolOptions{Name: "pool1"}
err := provision.AddPool(opts)
defer provision.RemovePool("pool1")
b := bytes.NewBufferString(`{"pool": "pool1", "id": "test1", "iaas": "another-test-iaas"}`)
req, err := http.NewRequest("POST", "/docker/node?register=false", b)
c.Assert(err, check.IsNil)
rec := httptest.NewRecorder()
err = addNodeHandler(rec, req, nil)
c.Assert(err, check.IsNil)
waitQueue()
nodes, err := mainDockerProvisioner.getCluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address, check.Equals, strings.TrimRight(server.URL(), "/"))
c.Assert(nodes[0].Metadata, check.DeepEquals, map[string]string{
"id": "test1",
"pool": "pool1",
"iaas": "another-test-iaas",
"iaas-id": "test1",
})
}
func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) {
defer func() {
machines, _ := iaas.ListMachines()
for _, m := range machines {
m.Destroy()
}
}()
iaas.RegisterIaasProvider("my-healer-iaas", newHealerIaaSConstructor("127.0.0.1", nil))
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaas.RegisterIaasProvider("my-healer-iaas", newHealerIaaSConstructor("localhost", nil))
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2.PrepareFailure("ping-failure", "/_ping")
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", urlPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
cluster, err := cluster.New(nil, &cluster.MapStorage{},
cluster.Node{Address: node1.URL()},
)
c.Assert(err, check.IsNil)
node1.PrepareFailure("pingErr", "/_ping")
cluster.StartActiveMonitoring(100 * time.Millisecond)
time.Sleep(300 * time.Millisecond)
cluster.StopActiveMonitoring()
var p dockerProvisioner
err = p.Initialize()
c.Assert(err, check.IsNil)
p.cluster = cluster
healer := nodeHealer{
locks: make(map[string]*sync.Mutex),
provisioner: &p,
disabledTime: 0,
failuresBeforeHealing: 1,
waitTimeNewMachine: 1 * time.Second,
}
nodes, err := p.getCluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1")
c.Assert(nodes[0].FailureCount() > 0, check.Equals, true)
nodes[0].Metadata["iaas"] = "my-healer-iaas"
created, err := healer.healNode(&nodes[0])
c.Assert(err, check.ErrorMatches, ".*error registering new node.*")
c.Assert(created.Address, check.Equals, "")
c.Assert(nodes[0].FailureCount(), check.Equals, 0)
nodes, err = p.getCluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL()))
c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1")
}
func (s *S) TestTemplateList(c *check.C) {
iaas.RegisterIaasProvider("ec2", newTestIaaS)
iaas.RegisterIaasProvider("other", newTestIaaS)
tpl1 := iaas.Template{
Name: "tpl1",
IaaSName: "ec2",
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "key1", Value: "val1"},
{Name: "key2", Value: "val2"},
}),
}
err := tpl1.Save()
c.Assert(err, check.IsNil)
defer iaas.DestroyTemplate("tpl1")
tpl2 := iaas.Template{
Name: "tpl2",
IaaSName: "other",
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "key4", Value: "valX"},
{Name: "key5", Value: "valY"},
}),
}
err = tpl2.Save()
c.Assert(err, check.IsNil)
defer iaas.DestroyTemplate("tpl2")
recorder := httptest.NewRecorder()
request, err := http.NewRequest("GET", "/iaas/templates", nil)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
c.Assert(recorder.Header().Get("Content-Type"), check.Equals, "application/json")
var templates []iaas.Template
err = json.Unmarshal(recorder.Body.Bytes(), &templates)
c.Assert(err, check.IsNil)
c.Assert(templates, check.HasLen, 2)
c.Assert(templates[0].Name, check.Equals, "tpl1")
c.Assert(templates[1].Name, check.Equals, "tpl2")
c.Assert(templates[0].IaaSName, check.Equals, "ec2")
c.Assert(templates[1].IaaSName, check.Equals, "other")
c.Assert(templates[0].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{
{Name: "key1", Value: "val1"},
{Name: "key2", Value: "val2"},
}))
c.Assert(templates[1].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{
{Name: "key4", Value: "valX"},
{Name: "key5", Value: "valY"},
}))
}
func (s *S) TestTemplateCreate(c *check.C) {
iaas.RegisterIaasProvider("my-iaas", newTestIaaS)
data := iaas.Template{
Name: "my-tpl",
IaaSName: "my-iaas",
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "x", Value: "y"},
{Name: "a", Value: "b"},
}),
}
bodyData, err := json.Marshal(data)
c.Assert(err, check.IsNil)
body := bytes.NewBuffer(bodyData)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("POST", "/iaas/templates", body)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusCreated)
defer iaas.DestroyTemplate("my-tpl")
templates, err := iaas.ListTemplates()
c.Assert(err, check.IsNil)
c.Assert(templates, check.HasLen, 1)
c.Assert(templates[0].Name, check.Equals, "my-tpl")
c.Assert(templates[0].IaaSName, check.Equals, "my-iaas")
c.Assert(templates[0].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{
{Name: "x", Value: "y"},
{Name: "a", Value: "b"},
}))
}
func (s *S) TestMachinesList(c *check.C) {
iaas.RegisterIaasProvider("test-iaas", newTestIaaS)
_, err := iaas.CreateMachineForIaaS("test-iaas", map[string]string{"id": "myid1"})
defer (&iaas.Machine{Id: "myid1"}).Destroy()
c.Assert(err, check.IsNil)
_, err = iaas.CreateMachineForIaaS("test-iaas", map[string]string{"id": "myid2"})
defer (&iaas.Machine{Id: "myid2"}).Destroy()
c.Assert(err, check.IsNil)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("GET", "/iaas/machines", nil)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
var machines []iaas.Machine
err = json.NewDecoder(recorder.Body).Decode(&machines)
c.Assert(err, check.IsNil)
c.Assert(machines[0].Id, check.Equals, "myid1")
c.Assert(machines[0].Address, check.Equals, "myid1.somewhere.com")
c.Assert(machines[0].CreationParams, check.DeepEquals, map[string]string{
"id": "myid1",
"iaas": "test-iaas",
"iaas-id": "myid1",
})
c.Assert(machines[1].Id, check.Equals, "myid2")
c.Assert(machines[1].Address, check.Equals, "myid2.somewhere.com")
c.Assert(machines[1].CreationParams, check.DeepEquals, map[string]string{
"id": "myid2",
"iaas": "test-iaas",
"iaas-id": "myid2",
})
}
func (s *S) TestHealerHandleErrorDoesntTriggerEventIfHealingCountTooLarge(c *check.C) {
nodes := []cluster.Node{
{Address: "addr1"}, {Address: "addr2"}, {Address: "addr3"}, {Address: "addr4"},
{Address: "addr5"}, {Address: "addr6"}, {Address: "addr7"}, {Address: "addr8"},
}
for i := 0; i < len(nodes)-1; i++ {
evt, err := NewHealingEvent(nodes[i])
c.Assert(err, check.IsNil)
err = evt.Update(nodes[i+1], nil)
c.Assert(err, check.IsNil)
}
iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("127.0.0.1", nil))
healer := NewNodeHealer(NodeHealerArgs{
DisabledTime: 20,
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
nodes[7].Metadata = map[string]string{
"Failures": "2",
"LastSuccess": "something",
"iaas": "my-healer-iaas",
}
waitTime := healer.HandleError(&nodes[7])
c.Assert(waitTime, check.Equals, time.Duration(20))
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 7)
}
func (s *HandlersSuite) TestAddNodeHandlerCreatingAnIaasMachine(c *gocheck.C) {
iaas.RegisterIaasProvider("test-iaas", TestIaaS{})
dCluster, _ = cluster.New(segregatedScheduler{}, &cluster.MapStorage{})
p := Pool{Name: "pool1"}
s.conn.Collection(schedulerCollection).Insert(p)
defer s.conn.Collection(schedulerCollection).RemoveId("pool1")
b := bytes.NewBufferString(`{"pool": "pool1", "id": "test1"}`)
req, err := http.NewRequest("POST", "/docker/node?register=false", b)
c.Assert(err, gocheck.IsNil)
rec := httptest.NewRecorder()
err = addNodeHandler(rec, req, nil)
c.Assert(err, gocheck.IsNil)
var result map[string]string
err = json.NewDecoder(rec.Body).Decode(&result)
c.Assert(err, gocheck.IsNil)
c.Assert(result, gocheck.DeepEquals, map[string]string{"description": "my iaas description"})
nodes, err := dCluster.Nodes()
c.Assert(err, gocheck.IsNil)
c.Assert(nodes, gocheck.HasLen, 1)
c.Assert(nodes[0].Address, gocheck.Equals, "http://test1.fake.host:1234")
c.Assert(nodes[0].Metadata, gocheck.DeepEquals, map[string]string{
"id": "test1",
"pool": "pool1",
"iaas": "test-iaas",
})
}
func (s *S) TestTemplateDestroy(c *check.C) {
iaas.RegisterIaasProvider("ec2", newTestIaaS)
tpl1 := iaas.Template{
Name: "tpl1",
IaaSName: "ec2",
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "key1", Value: "val1"},
{Name: "key2", Value: "val2"},
}),
}
err := tpl1.Save()
c.Assert(err, check.IsNil)
defer iaas.DestroyTemplate("tpl1")
recorder := httptest.NewRecorder()
request, err := http.NewRequest("DELETE", "/iaas/templates/tpl1", nil)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
templates, err := iaas.ListTemplates()
c.Assert(err, check.IsNil)
c.Assert(templates, check.HasLen, 0)
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: event.TargetTypeIaas, Value: "ec2"},
Owner: s.token.GetUserName(),
Kind: "machine.template.delete",
StartCustomData: []map[string]interface{}{
{"name": ":template_name", "value": "tpl1"},
},
}, eventtest.HasEvent)
}
func (s *S) TestHealerHandleErrorThrottled(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
node, err := p.GetNode("http://addr1:1")
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
healer.started = time.Now().Add(-3 * time.Second)
conf := healerConfig()
err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)})
c.Assert(err, check.IsNil)
err = healer.UpdateNodeData(node, []provision.NodeCheckResult{})
c.Assert(err, check.IsNil)
time.Sleep(1200 * time.Millisecond)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
for i := 0; i < 3; i++ {
var evt *event.Event
evt, err = event.NewInternal(&event.Opts{
Target: event.Target{Type: "node", Value: nodes[0].Address()},
InternalKind: "healer",
Allowed: event.Allowed(permission.PermPoolReadEvents),
})
c.Assert(err, check.IsNil)
err = evt.Done(nil)
c.Assert(err, check.IsNil)
}
err = healer.tryHealingNode(nodes[0], "myreason", nil)
c.Assert(err, check.ErrorMatches, "Error trying to insert node healing event, healing aborted: event throttled, limit for healer on node \".*?\" is 3 every 5m0s")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
}
func (s *S) TestTemplateUpdateBadRequest(c *check.C) {
iaas.RegisterIaasProvider("my-iaas", newTestIaaS)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", nil)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusBadRequest)
}
func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) {
iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("127.0.0.1", nil))
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("localhost", nil))
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
node2.PrepareFailure("ping-failure", "/_ping")
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", dockertest.URLPort(node2.URL()))
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p, err := s.newFakeDockerProvisioner(node1.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
node1.PrepareFailure("pingErr", "/_ping")
p.Cluster().StartActiveMonitoring(100 * time.Millisecond)
time.Sleep(300 * time.Millisecond)
p.Cluster().StopActiveMonitoring()
healer := NewNodeHealer(NodeHealerArgs{
Provisioner: p,
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Second,
})
nodes, err := p.Cluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1")
c.Assert(nodes[0].FailureCount() > 0, check.Equals, true)
nodes[0].Metadata["iaas"] = "my-healer-iaas"
created, err := healer.healNode(&nodes[0])
c.Assert(err, check.ErrorMatches, ".*timeout waiting for result.*")
c.Assert(created.Address, check.Equals, "")
c.Assert(nodes[0].FailureCount(), check.Equals, 0)
nodes, err = p.Cluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1")
}
func (s *S) TestTemplateUpdate(c *check.C) {
iaas.RegisterIaasProvider("my-iaas", newTestIaaS)
tpl1 := iaas.Template{
Name: "my-tpl",
IaaSName: "my-iaas",
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "x", Value: "y"},
{Name: "a", Value: "b"},
}),
}
err := tpl1.Save()
c.Assert(err, check.IsNil)
tplParam := iaas.Template{
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "x", Value: ""},
{Name: "y", Value: "8"},
{Name: "z", Value: "9"},
}),
}
v, err := form.EncodeToValues(&tplParam)
c.Assert(err, check.IsNil)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", strings.NewReader(v.Encode()))
c.Assert(err, check.IsNil)
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
templates, err := iaas.ListTemplates()
c.Assert(err, check.IsNil)
c.Assert(templates, check.HasLen, 1)
c.Assert(templates[0].Name, check.Equals, "my-tpl")
c.Assert(templates[0].IaaSName, check.Equals, "my-iaas")
sort.Sort(templates[0].Data)
c.Assert(templates[0].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{
{Name: "a", Value: "b"},
{Name: "y", Value: "8"},
{Name: "z", Value: "9"},
}))
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: event.TargetTypeIaas, Value: "my-iaas"},
Owner: s.token.GetUserName(),
Kind: "machine.template.update",
StartCustomData: []map[string]interface{}{
{"name": ":template_name", "value": "my-tpl"},
{"name": "Data.0.Name", "value": "x"},
{"name": "Data.0.Value", "value": ""},
{"name": "Data.1.Name", "value": "y"},
{"name": "Data.1.Value", "value": "8"},
{"name": "Data.2.Name", "value": "z"},
{"name": "Data.2.Value", "value": "9"},
},
}, eventtest.HasEvent)
}
func (s *S) TestMachinesDestroy(c *check.C) {
iaas.RegisterIaasProvider("test-iaas", newTestIaaS)
_, err := iaas.CreateMachineForIaaS("test-iaas", map[string]string{"id": "myid1"})
c.Assert(err, check.IsNil)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("DELETE", "/iaas/machines/myid1", nil)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
}
func (s *S) TestHealerHealNodeDestroyError(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaasInst.DelErr = fmt.Errorf("my destroy error")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
buf := bytes.Buffer{}
log.SetLogger(log.NewWriterLogger(&buf, false))
defer log.SetLogger(nil)
created, err := healer.healNode(nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, "http://addr2:2")
c.Assert(buf.String(), check.Matches, "(?s).*my destroy error.*")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
}
func (s *S) TestAddNodeHandlerCreatingAnIaasMachine(c *check.C) {
iaas.RegisterIaasProvider("test-iaas", newTestIaaS)
opts := provision.AddPoolOptions{Name: "pool1"}
err := provision.AddPool(opts)
c.Assert(err, check.IsNil)
defer provision.RemovePool("pool1")
params := provision.AddNodeOptions{
Register: false,
Metadata: map[string]string{
"id": "test1",
"pool": "pool1",
"iaas": "test-iaas",
},
}
v, err := form.EncodeToValues(¶ms)
c.Assert(err, check.IsNil)
b := strings.NewReader(v.Encode())
req, err := http.NewRequest("POST", "/node", b)
c.Assert(err, check.IsNil)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Authorization", s.token.GetValue())
rec := httptest.NewRecorder()
m := RunServer(true)
m.ServeHTTP(rec, req)
c.Assert(rec.Code, check.Equals, http.StatusCreated)
c.Assert(rec.Body.String(), check.Equals, "")
nodes, err := s.provisioner.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://test1.somewhere.com:2375")
c.Assert(nodes[0].Metadata(), check.DeepEquals, map[string]string{
"id": "test1",
"pool": "pool1",
"iaas": "test-iaas",
"iaas-id": "test1",
})
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: event.TargetTypeNode, Value: "http://test1.somewhere.com:2375"},
Owner: s.token.GetUserName(),
Kind: "node.create",
StartCustomData: []map[string]interface{}{
{"name": "Metadata.id", "value": "test1"},
{"name": "Metadata.pool", "value": "pool1"},
{"name": "Register", "value": ""},
},
}, eventtest.HasEvent)
}
func (s *S) TestHealerHealNode(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
created, err := healer.healNode(nodes[0])
c.Assert(err, check.IsNil)
c.Assert(created.Address, check.Equals, "http://addr2:2")
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2")
machines, err = iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr2")
}
func (s *HandlersSuite) TestRemoveNodeHandlerRemoveIaaS(c *check.C) {
iaas.RegisterIaasProvider("my-xxx-iaas", newTestIaaS)
machine, err := iaas.CreateMachineForIaaS("my-xxx-iaas", map[string]string{})
c.Assert(err, check.IsNil)
mainDockerProvisioner.cluster, err = cluster.New(nil, &cluster.MapStorage{})
c.Assert(err, check.IsNil)
_, err = mainDockerProvisioner.getCluster().Register(fmt.Sprintf("http://%s:2375", machine.Address), nil)
c.Assert(err, check.IsNil)
b := bytes.NewBufferString(fmt.Sprintf(`{"address": "http://%s:2375", "remove_iaas": "true"}`, machine.Address))
req, err := http.NewRequest("POST", "/node/remove", b)
c.Assert(err, check.IsNil)
rec := httptest.NewRecorder()
err = removeNodeHandler(rec, req, nil)
c.Assert(err, check.IsNil)
nodes, err := mainDockerProvisioner.getCluster().Nodes()
c.Assert(len(nodes), check.Equals, 0)
_, err = iaas.FindMachineById(machine.Id)
c.Assert(err, check.Equals, mgo.ErrNotFound)
}
func (s *HandlersSuite) TestRemoveNodeHandlerWithoutRemoveIaaS(c *gocheck.C) {
iaas.RegisterIaasProvider("some-iaas", TestIaaS{})
machine, err := iaas.CreateMachineForIaaS("some-iaas", map[string]string{})
c.Assert(err, gocheck.IsNil)
dCluster, err = cluster.New(nil, &cluster.MapStorage{})
c.Assert(err, gocheck.IsNil)
err = dCluster.Register(fmt.Sprintf("http://%s:4243", machine.Address), nil)
c.Assert(err, gocheck.IsNil)
b := bytes.NewBufferString(fmt.Sprintf(`{"address": "http://%s:4243", "remove_iaas": "false"}`, machine.Address))
req, err := http.NewRequest("POST", "/node/remove", b)
c.Assert(err, gocheck.IsNil)
rec := httptest.NewRecorder()
err = removeNodeHandler(rec, req, nil)
c.Assert(err, gocheck.IsNil)
nodes, err := dCluster.Nodes()
c.Assert(len(nodes), gocheck.Equals, 0)
dbM, err := iaas.FindMachineById(machine.Id)
c.Assert(err, gocheck.IsNil)
c.Assert(dbM.Id, gocheck.Equals, machine.Id)
}
func (s *S) TestMachinesDestroy(c *check.C) {
iaas.RegisterIaasProvider("test-iaas", newTestIaaS)
_, err := iaas.CreateMachineForIaaS("test-iaas", map[string]string{"id": "myid1"})
c.Assert(err, check.IsNil)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("DELETE", "/iaas/machines/myid1", nil)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: event.TargetTypeIaas, Value: "test-iaas"},
Owner: s.token.GetUserName(),
Kind: "machine.delete",
StartCustomData: []map[string]interface{}{
{"name": ":machine_id", "value": "myid1"},
},
}, eventtest.HasEvent)
}
func (s *S) TestTemplateUpdate(c *check.C) {
iaas.RegisterIaasProvider("my-iaas", newTestIaaS)
tpl1 := iaas.Template{
Name: "my-tpl",
IaaSName: "my-iaas",
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "x", Value: "y"},
{Name: "a", Value: "b"},
}),
}
err := tpl1.Save()
c.Assert(err, check.IsNil)
tplParam := iaas.Template{
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "x", Value: ""},
{Name: "y", Value: "8"},
{Name: "z", Value: "9"},
}),
}
bodyData, err := json.Marshal(tplParam)
c.Assert(err, check.IsNil)
body := bytes.NewBuffer(bodyData)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", body)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
templates, err := iaas.ListTemplates()
c.Assert(err, check.IsNil)
c.Assert(templates, check.HasLen, 1)
c.Assert(templates[0].Name, check.Equals, "my-tpl")
c.Assert(templates[0].IaaSName, check.Equals, "my-iaas")
sort.Sort(templates[0].Data)
c.Assert(templates[0].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{
{Name: "a", Value: "b"},
{Name: "y", Value: "8"},
{Name: "z", Value: "9"},
}))
}
func (s *S) TestTemplateUpdateNotFound(c *check.C) {
iaas.RegisterIaasProvider("my-iaas", newTestIaaS)
tplParam := iaas.Template{
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "x", Value: ""},
{Name: "y", Value: "8"},
{Name: "z", Value: "9"},
}),
}
v, err := form.EncodeToValues(&tplParam)
c.Assert(err, check.IsNil)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", strings.NewReader(v.Encode()))
c.Assert(err, check.IsNil)
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusNotFound)
c.Assert(recorder.Body.String(), check.Equals, "template not found\n")
}
func (s *S) TestTemplateUpdateNotFound(c *check.C) {
iaas.RegisterIaasProvider("my-iaas", newTestIaaS)
tplParam := iaas.Template{
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "x", Value: ""},
{Name: "y", Value: "8"},
{Name: "z", Value: "9"},
}),
}
bodyData, err := json.Marshal(tplParam)
c.Assert(err, check.IsNil)
body := bytes.NewBuffer(bodyData)
recorder := httptest.NewRecorder()
request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", body)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusNotFound)
c.Assert(recorder.Body.String(), check.Equals, "template not found\n")
}
func (s *S) TestTryHealingNodeDoubleCheck(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
config.Set("iaas:node-protocol", "http")
config.Set("iaas:node-port", 2)
defer config.Unset("iaas:node-protocol")
defer config.Unset("iaas:node-port")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.started = time.Now().Add(-3 * time.Second)
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
healErr := healer.tryHealingNode(nodes[0], "something", nil)
c.Assert(healErr, check.IsNil)
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
machines, err := iaas.ListMachines()
c.Assert(err, check.IsNil)
c.Assert(machines, check.HasLen, 1)
c.Assert(machines[0].Address, check.Equals, "addr1")
c.Assert(eventtest.EventDesc{
IsEmpty: true,
}, eventtest.HasEvent)
}
func (s *S) TestHealerHealNodeCreateMachineError(c *check.C) {
factory, iaasInst := dockertest.NewHealerIaaSConstructorWithInst("127.0.0.1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "localhost"
iaasInst.Err = fmt.Errorf("my create machine error")
node1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
p, err := s.newFakeDockerProvisioner(node1.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
node1.PrepareFailure("pingErr", "/_ping")
p.Cluster().StartActiveMonitoring(100 * time.Millisecond)
time.Sleep(300 * time.Millisecond)
p.Cluster().StopActiveMonitoring()
healer := NewNodeHealer(NodeHealerArgs{
Provisioner: p,
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
nodes, err := p.Cluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1")
c.Assert(nodes[0].FailureCount() > 0, check.Equals, true)
nodes[0].Metadata["iaas"] = "my-healer-iaas"
created, err := healer.healNode(&nodes[0])
c.Assert(err, check.ErrorMatches, ".*my create machine error.*")
c.Assert(created.Address, check.Equals, "")
c.Assert(nodes[0].FailureCount(), check.Equals, 0)
nodes, err = p.Cluster().UnfilteredNodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL()))
c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1")
}
func (s *S) TestRemoveNodeHandlerWithRemoveIaaS(c *check.C) {
iaas.RegisterIaasProvider("some-iaas", newTestIaaS)
machine, err := iaas.CreateMachineForIaaS("some-iaas", map[string]string{"id": "m1"})
c.Assert(err, check.IsNil)
err = s.provisioner.AddNode(provision.AddNodeOptions{
Address: fmt.Sprintf("http://%s:2375", machine.Address),
})
c.Assert(err, check.IsNil)
u := fmt.Sprintf("/node/http://%s:2375?remove-iaas=true", machine.Address)
req, err := http.NewRequest("DELETE", u, nil)
c.Assert(err, check.IsNil)
req.Header.Set("Authorization", "bearer "+s.token.GetValue())
rec := httptest.NewRecorder()
server := RunServer(true)
server.ServeHTTP(rec, req)
c.Assert(rec.Body.String(), check.Equals, "rebalancing...remove done!")
c.Assert(rec.Code, check.Equals, http.StatusOK)
nodes, err := s.provisioner.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 0)
_, err = iaas.FindMachineById(machine.Id)
c.Assert(err, check.Equals, mgo.ErrNotFound)
}
func (s *S) TestHealerHealNodeCreateMachineError(c *check.C) {
factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1")
iaas.RegisterIaasProvider("my-healer-iaas", factory)
_, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{})
c.Assert(err, check.IsNil)
iaasInst.Addr = "addr2"
iaasInst.Err = fmt.Errorf("my create machine error")
p := provisiontest.ProvisionerInstance
err = p.AddNode(provision.AddNodeOptions{
Address: "http://addr1:1",
Metadata: map[string]string{"iaas": "my-healer-iaas"},
})
c.Assert(err, check.IsNil)
healer := newNodeHealer(nodeHealerArgs{
FailuresBeforeHealing: 1,
WaitTimeNewMachine: time.Minute,
})
healer.Shutdown()
nodes, err := p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
fakeNode := nodes[0].(*provisiontest.FakeNode)
fakeNode.SetHealth(1, false)
c.Assert(fakeNode.FailureCount() > 0, check.Equals, true)
created, err := healer.healNode(nodes[0])
c.Assert(err, check.ErrorMatches, ".*my create machine error.*")
c.Assert(created, check.IsNil)
c.Assert(fakeNode.FailureCount(), check.Equals, 0)
nodes, err = p.ListNodes(nil)
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 1)
c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1")
c.Assert(nodes[0].Status(), check.Equals, "enabled")
}
func (s *S) TestTemplateDestroy(c *check.C) {
iaas.RegisterIaasProvider("ec2", newTestIaaS)
tpl1 := iaas.Template{
Name: "tpl1",
IaaSName: "ec2",
Data: iaas.TemplateDataList([]iaas.TemplateData{
{Name: "key1", Value: "val1"},
{Name: "key2", Value: "val2"},
}),
}
err := tpl1.Save()
c.Assert(err, check.IsNil)
defer iaas.DestroyTemplate("tpl1")
recorder := httptest.NewRecorder()
request, err := http.NewRequest("DELETE", "/iaas/templates/tpl1", nil)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue())
m := RunServer(true)
m.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
templates, err := iaas.ListTemplates()
c.Assert(err, check.IsNil)
c.Assert(templates, check.HasLen, 0)
}
func init() {
iaas.RegisterIaasProvider("digitalocean", newDigitalOceanIaas)
}
func init() {
iaas.RegisterIaasProvider("cloudstack", &CloudstackIaaS{})
}
