func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) { iaas.RegisterIaasProvider("my-healer-iaas", iaasTesting.NewHealerIaaSConstructor("addr1", nil)) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaas.RegisterIaasProvider("my-healer-iaas", iaasTesting.NewHealerIaaSConstructor("addr2", nil)) config.Set("iaas:node-protocol", "http") config.Set("iaas:node-port", 2) defer config.Unset("iaas:node-protocol") defer config.Unset("iaas:node-port") p := provisiontest.ProvisionerInstance err = p.AddNode(provision.AddNodeOptions{ Address: "http://addr1:1", Metadata: map[string]string{"iaas": "my-healer-iaas"}, }) c.Assert(err, check.IsNil) p.PrepareFailure("AddNode", fmt.Errorf("add node error")) healer := newNodeHealer(nodeHealerArgs{ WaitTimeNewMachine: time.Second, }) healer.Shutdown() nodes, err := p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") created, err := healer.healNode(nodes[0]) c.Assert(err, check.ErrorMatches, ".*error registering new node: add node error.*") c.Assert(created, check.IsNil) nodes, err = p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") c.Assert(nodes[0].Status(), check.Equals, "enabled") }
func (s *HandlersSuite) TestAddNodeHandlerCreatingAnIaasMachineExplicit(c *check.C) { server, waitQueue := s.startFakeDockerNode(c) defer server.Stop() iaas.RegisterIaasProvider("test-iaas", newTestIaaS) iaas.RegisterIaasProvider("another-test-iaas", newTestIaaS) mainDockerProvisioner.cluster, _ = cluster.New(&segregatedScheduler{}, &cluster.MapStorage{}) opts := provision.AddPoolOptions{Name: "pool1"} err := provision.AddPool(opts) defer provision.RemovePool("pool1") b := bytes.NewBufferString(`{"pool": "pool1", "id": "test1", "iaas": "another-test-iaas"}`) req, err := http.NewRequest("POST", "/docker/node?register=false", b) c.Assert(err, check.IsNil) rec := httptest.NewRecorder() err = addNodeHandler(rec, req, nil) c.Assert(err, check.IsNil) waitQueue() nodes, err := mainDockerProvisioner.getCluster().Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address, check.Equals, strings.TrimRight(server.URL(), "/")) c.Assert(nodes[0].Metadata, check.DeepEquals, map[string]string{ "id": "test1", "pool": "pool1", "iaas": "another-test-iaas", "iaas-id": "test1", }) }
func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) { defer func() { machines, _ := iaas.ListMachines() for _, m := range machines { m.Destroy() } }() iaas.RegisterIaasProvider("my-healer-iaas", newHealerIaaSConstructor("127.0.0.1", nil)) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaas.RegisterIaasProvider("my-healer-iaas", newHealerIaaSConstructor("localhost", nil)) node1, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) node2, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) node2.PrepareFailure("ping-failure", "/_ping") config.Set("iaas:node-protocol", "http") config.Set("iaas:node-port", urlPort(node2.URL())) defer config.Unset("iaas:node-protocol") defer config.Unset("iaas:node-port") cluster, err := cluster.New(nil, &cluster.MapStorage{}, cluster.Node{Address: node1.URL()}, ) c.Assert(err, check.IsNil) node1.PrepareFailure("pingErr", "/_ping") cluster.StartActiveMonitoring(100 * time.Millisecond) time.Sleep(300 * time.Millisecond) cluster.StopActiveMonitoring() var p dockerProvisioner err = p.Initialize() c.Assert(err, check.IsNil) p.cluster = cluster healer := nodeHealer{ locks: make(map[string]*sync.Mutex), provisioner: &p, disabledTime: 0, failuresBeforeHealing: 1, waitTimeNewMachine: 1 * time.Second, } nodes, err := p.getCluster().UnfilteredNodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL())) c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1") c.Assert(nodes[0].FailureCount() > 0, check.Equals, true) nodes[0].Metadata["iaas"] = "my-healer-iaas" created, err := healer.healNode(&nodes[0]) c.Assert(err, check.ErrorMatches, ".*error registering new node.*") c.Assert(created.Address, check.Equals, "") c.Assert(nodes[0].FailureCount(), check.Equals, 0) nodes, err = p.getCluster().UnfilteredNodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(urlPort(nodes[0].Address), check.Equals, urlPort(node1.URL())) c.Assert(urlToHost(nodes[0].Address), check.Equals, "127.0.0.1") }
func (s *S) TestTemplateList(c *check.C) { iaas.RegisterIaasProvider("ec2", newTestIaaS) iaas.RegisterIaasProvider("other", newTestIaaS) tpl1 := iaas.Template{ Name: "tpl1", IaaSName: "ec2", Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "key1", Value: "val1"}, {Name: "key2", Value: "val2"}, }), } err := tpl1.Save() c.Assert(err, check.IsNil) defer iaas.DestroyTemplate("tpl1") tpl2 := iaas.Template{ Name: "tpl2", IaaSName: "other", Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "key4", Value: "valX"}, {Name: "key5", Value: "valY"}, }), } err = tpl2.Save() c.Assert(err, check.IsNil) defer iaas.DestroyTemplate("tpl2") recorder := httptest.NewRecorder() request, err := http.NewRequest("GET", "/iaas/templates", nil) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.token.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusOK) c.Assert(recorder.Header().Get("Content-Type"), check.Equals, "application/json") var templates []iaas.Template err = json.Unmarshal(recorder.Body.Bytes(), &templates) c.Assert(err, check.IsNil) c.Assert(templates, check.HasLen, 2) c.Assert(templates[0].Name, check.Equals, "tpl1") c.Assert(templates[1].Name, check.Equals, "tpl2") c.Assert(templates[0].IaaSName, check.Equals, "ec2") c.Assert(templates[1].IaaSName, check.Equals, "other") c.Assert(templates[0].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{ {Name: "key1", Value: "val1"}, {Name: "key2", Value: "val2"}, })) c.Assert(templates[1].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{ {Name: "key4", Value: "valX"}, {Name: "key5", Value: "valY"}, })) }
func (s *S) TestTemplateCreate(c *check.C) { iaas.RegisterIaasProvider("my-iaas", newTestIaaS) data := iaas.Template{ Name: "my-tpl", IaaSName: "my-iaas", Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "x", Value: "y"}, {Name: "a", Value: "b"}, }), } bodyData, err := json.Marshal(data) c.Assert(err, check.IsNil) body := bytes.NewBuffer(bodyData) recorder := httptest.NewRecorder() request, err := http.NewRequest("POST", "/iaas/templates", body) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusCreated) defer iaas.DestroyTemplate("my-tpl") templates, err := iaas.ListTemplates() c.Assert(err, check.IsNil) c.Assert(templates, check.HasLen, 1) c.Assert(templates[0].Name, check.Equals, "my-tpl") c.Assert(templates[0].IaaSName, check.Equals, "my-iaas") c.Assert(templates[0].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{ {Name: "x", Value: "y"}, {Name: "a", Value: "b"}, })) }
func (s *S) TestMachinesList(c *check.C) { iaas.RegisterIaasProvider("test-iaas", newTestIaaS) _, err := iaas.CreateMachineForIaaS("test-iaas", map[string]string{"id": "myid1"}) defer (&iaas.Machine{Id: "myid1"}).Destroy() c.Assert(err, check.IsNil) _, err = iaas.CreateMachineForIaaS("test-iaas", map[string]string{"id": "myid2"}) defer (&iaas.Machine{Id: "myid2"}).Destroy() c.Assert(err, check.IsNil) recorder := httptest.NewRecorder() request, err := http.NewRequest("GET", "/iaas/machines", nil) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusOK) var machines []iaas.Machine err = json.NewDecoder(recorder.Body).Decode(&machines) c.Assert(err, check.IsNil) c.Assert(machines[0].Id, check.Equals, "myid1") c.Assert(machines[0].Address, check.Equals, "myid1.somewhere.com") c.Assert(machines[0].CreationParams, check.DeepEquals, map[string]string{ "id": "myid1", "iaas": "test-iaas", "iaas-id": "myid1", }) c.Assert(machines[1].Id, check.Equals, "myid2") c.Assert(machines[1].Address, check.Equals, "myid2.somewhere.com") c.Assert(machines[1].CreationParams, check.DeepEquals, map[string]string{ "id": "myid2", "iaas": "test-iaas", "iaas-id": "myid2", }) }
func (s *S) TestHealerHandleErrorDoesntTriggerEventIfHealingCountTooLarge(c *check.C) { nodes := []cluster.Node{ {Address: "addr1"}, {Address: "addr2"}, {Address: "addr3"}, {Address: "addr4"}, {Address: "addr5"}, {Address: "addr6"}, {Address: "addr7"}, {Address: "addr8"}, } for i := 0; i < len(nodes)-1; i++ { evt, err := NewHealingEvent(nodes[i]) c.Assert(err, check.IsNil) err = evt.Update(nodes[i+1], nil) c.Assert(err, check.IsNil) } iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("127.0.0.1", nil)) healer := NewNodeHealer(NodeHealerArgs{ DisabledTime: 20, FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Minute, }) nodes[7].Metadata = map[string]string{ "Failures": "2", "LastSuccess": "something", "iaas": "my-healer-iaas", } waitTime := healer.HandleError(&nodes[7]) c.Assert(waitTime, check.Equals, time.Duration(20)) healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() var events []HealingEvent err = healingColl.Find(nil).All(&events) c.Assert(err, check.IsNil) c.Assert(events, check.HasLen, 7) }
func (s *HandlersSuite) TestAddNodeHandlerCreatingAnIaasMachine(c *gocheck.C) { iaas.RegisterIaasProvider("test-iaas", TestIaaS{}) dCluster, _ = cluster.New(segregatedScheduler{}, &cluster.MapStorage{}) p := Pool{Name: "pool1"} s.conn.Collection(schedulerCollection).Insert(p) defer s.conn.Collection(schedulerCollection).RemoveId("pool1") b := bytes.NewBufferString(`{"pool": "pool1", "id": "test1"}`) req, err := http.NewRequest("POST", "/docker/node?register=false", b) c.Assert(err, gocheck.IsNil) rec := httptest.NewRecorder() err = addNodeHandler(rec, req, nil) c.Assert(err, gocheck.IsNil) var result map[string]string err = json.NewDecoder(rec.Body).Decode(&result) c.Assert(err, gocheck.IsNil) c.Assert(result, gocheck.DeepEquals, map[string]string{"description": "my iaas description"}) nodes, err := dCluster.Nodes() c.Assert(err, gocheck.IsNil) c.Assert(nodes, gocheck.HasLen, 1) c.Assert(nodes[0].Address, gocheck.Equals, "http://test1.fake.host:1234") c.Assert(nodes[0].Metadata, gocheck.DeepEquals, map[string]string{ "id": "test1", "pool": "pool1", "iaas": "test-iaas", }) }
func (s *S) TestTemplateDestroy(c *check.C) { iaas.RegisterIaasProvider("ec2", newTestIaaS) tpl1 := iaas.Template{ Name: "tpl1", IaaSName: "ec2", Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "key1", Value: "val1"}, {Name: "key2", Value: "val2"}, }), } err := tpl1.Save() c.Assert(err, check.IsNil) defer iaas.DestroyTemplate("tpl1") recorder := httptest.NewRecorder() request, err := http.NewRequest("DELETE", "/iaas/templates/tpl1", nil) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.token.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusOK) templates, err := iaas.ListTemplates() c.Assert(err, check.IsNil) c.Assert(templates, check.HasLen, 0) c.Assert(eventtest.EventDesc{ Target: event.Target{Type: event.TargetTypeIaas, Value: "ec2"}, Owner: s.token.GetUserName(), Kind: "machine.template.delete", StartCustomData: []map[string]interface{}{ {"name": ":template_name", "value": "tpl1"}, }, }, eventtest.HasEvent) }
func (s *S) TestHealerHandleErrorThrottled(c *check.C) { factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1") iaas.RegisterIaasProvider("my-healer-iaas", factory) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaasInst.Addr = "addr2" config.Set("iaas:node-protocol", "http") config.Set("iaas:node-port", 2) defer config.Unset("iaas:node-protocol") defer config.Unset("iaas:node-port") p := provisiontest.ProvisionerInstance err = p.AddNode(provision.AddNodeOptions{ Address: "http://addr1:1", Metadata: map[string]string{"iaas": "my-healer-iaas"}, }) c.Assert(err, check.IsNil) node, err := p.GetNode("http://addr1:1") c.Assert(err, check.IsNil) healer := newNodeHealer(nodeHealerArgs{ FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Minute, }) healer.Shutdown() healer.started = time.Now().Add(-3 * time.Second) conf := healerConfig() err = conf.SaveBase(NodeHealerConfig{Enabled: boolPtr(true), MaxUnresponsiveTime: intPtr(1)}) c.Assert(err, check.IsNil) err = healer.UpdateNodeData(node, []provision.NodeCheckResult{}) c.Assert(err, check.IsNil) time.Sleep(1200 * time.Millisecond) nodes, err := p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") machines, err := iaas.ListMachines() c.Assert(err, check.IsNil) c.Assert(machines, check.HasLen, 1) c.Assert(machines[0].Address, check.Equals, "addr1") for i := 0; i < 3; i++ { var evt *event.Event evt, err = event.NewInternal(&event.Opts{ Target: event.Target{Type: "node", Value: nodes[0].Address()}, InternalKind: "healer", Allowed: event.Allowed(permission.PermPoolReadEvents), }) c.Assert(err, check.IsNil) err = evt.Done(nil) c.Assert(err, check.IsNil) } err = healer.tryHealingNode(nodes[0], "myreason", nil) c.Assert(err, check.ErrorMatches, "Error trying to insert node healing event, healing aborted: event throttled, limit for healer on node \".*?\" is 3 every 5m0s") nodes, err = p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") }
func (s *S) TestTemplateUpdateBadRequest(c *check.C) { iaas.RegisterIaasProvider("my-iaas", newTestIaaS) recorder := httptest.NewRecorder() request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", nil) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.token.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusBadRequest) }
func (s *S) TestHealerHealNodeWaitAndRegisterError(c *check.C) { iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("127.0.0.1", nil)) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaas.RegisterIaasProvider("my-healer-iaas", dockertest.NewHealerIaaSConstructor("localhost", nil)) node1, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) node2, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) node2.PrepareFailure("ping-failure", "/_ping") config.Set("iaas:node-protocol", "http") config.Set("iaas:node-port", dockertest.URLPort(node2.URL())) defer config.Unset("iaas:node-protocol") defer config.Unset("iaas:node-port") p, err := s.newFakeDockerProvisioner(node1.URL()) c.Assert(err, check.IsNil) defer p.Destroy() node1.PrepareFailure("pingErr", "/_ping") p.Cluster().StartActiveMonitoring(100 * time.Millisecond) time.Sleep(300 * time.Millisecond) p.Cluster().StopActiveMonitoring() healer := NewNodeHealer(NodeHealerArgs{ Provisioner: p, FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Second, }) nodes, err := p.Cluster().UnfilteredNodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL())) c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1") c.Assert(nodes[0].FailureCount() > 0, check.Equals, true) nodes[0].Metadata["iaas"] = "my-healer-iaas" created, err := healer.healNode(&nodes[0]) c.Assert(err, check.ErrorMatches, ".*timeout waiting for result.*") c.Assert(created.Address, check.Equals, "") c.Assert(nodes[0].FailureCount(), check.Equals, 0) nodes, err = p.Cluster().Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL())) c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1") }
func (s *S) TestTemplateUpdate(c *check.C) { iaas.RegisterIaasProvider("my-iaas", newTestIaaS) tpl1 := iaas.Template{ Name: "my-tpl", IaaSName: "my-iaas", Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "x", Value: "y"}, {Name: "a", Value: "b"}, }), } err := tpl1.Save() c.Assert(err, check.IsNil) tplParam := iaas.Template{ Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "x", Value: ""}, {Name: "y", Value: "8"}, {Name: "z", Value: "9"}, }), } v, err := form.EncodeToValues(&tplParam) c.Assert(err, check.IsNil) recorder := httptest.NewRecorder() request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", strings.NewReader(v.Encode())) c.Assert(err, check.IsNil) request.Header.Set("Content-Type", "application/x-www-form-urlencoded") request.Header.Set("Authorization", "bearer "+s.token.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusOK) templates, err := iaas.ListTemplates() c.Assert(err, check.IsNil) c.Assert(templates, check.HasLen, 1) c.Assert(templates[0].Name, check.Equals, "my-tpl") c.Assert(templates[0].IaaSName, check.Equals, "my-iaas") sort.Sort(templates[0].Data) c.Assert(templates[0].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{ {Name: "a", Value: "b"}, {Name: "y", Value: "8"}, {Name: "z", Value: "9"}, })) c.Assert(eventtest.EventDesc{ Target: event.Target{Type: event.TargetTypeIaas, Value: "my-iaas"}, Owner: s.token.GetUserName(), Kind: "machine.template.update", StartCustomData: []map[string]interface{}{ {"name": ":template_name", "value": "my-tpl"}, {"name": "Data.0.Name", "value": "x"}, {"name": "Data.0.Value", "value": ""}, {"name": "Data.1.Name", "value": "y"}, {"name": "Data.1.Value", "value": "8"}, {"name": "Data.2.Name", "value": "z"}, {"name": "Data.2.Value", "value": "9"}, }, }, eventtest.HasEvent) }
func (s *S) TestMachinesDestroy(c *check.C) { iaas.RegisterIaasProvider("test-iaas", newTestIaaS) _, err := iaas.CreateMachineForIaaS("test-iaas", map[string]string{"id": "myid1"}) c.Assert(err, check.IsNil) recorder := httptest.NewRecorder() request, err := http.NewRequest("DELETE", "/iaas/machines/myid1", nil) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusOK) }
func (s *S) TestHealerHealNodeDestroyError(c *check.C) { factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1") iaasInst.DelErr = fmt.Errorf("my destroy error") iaas.RegisterIaasProvider("my-healer-iaas", factory) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaasInst.Addr = "addr2" config.Set("iaas:node-protocol", "http") config.Set("iaas:node-port", 2) defer config.Unset("iaas:node-protocol") defer config.Unset("iaas:node-port") p := provisiontest.ProvisionerInstance err = p.AddNode(provision.AddNodeOptions{ Address: "http://addr1:1", Metadata: map[string]string{"iaas": "my-healer-iaas"}, }) c.Assert(err, check.IsNil) healer := newNodeHealer(nodeHealerArgs{ WaitTimeNewMachine: time.Minute, }) healer.Shutdown() nodes, err := p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") machines, err := iaas.ListMachines() c.Assert(err, check.IsNil) c.Assert(machines, check.HasLen, 1) c.Assert(machines[0].Address, check.Equals, "addr1") buf := bytes.Buffer{} log.SetLogger(log.NewWriterLogger(&buf, false)) defer log.SetLogger(nil) created, err := healer.healNode(nodes[0]) c.Assert(err, check.IsNil) c.Assert(created.Address, check.Equals, "http://addr2:2") c.Assert(buf.String(), check.Matches, "(?s).*my destroy error.*") nodes, err = p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2") machines, err = iaas.ListMachines() c.Assert(err, check.IsNil) c.Assert(machines, check.HasLen, 1) c.Assert(machines[0].Address, check.Equals, "addr2") }
func (s *S) TestAddNodeHandlerCreatingAnIaasMachine(c *check.C) { iaas.RegisterIaasProvider("test-iaas", newTestIaaS) opts := provision.AddPoolOptions{Name: "pool1"} err := provision.AddPool(opts) c.Assert(err, check.IsNil) defer provision.RemovePool("pool1") params := provision.AddNodeOptions{ Register: false, Metadata: map[string]string{ "id": "test1", "pool": "pool1", "iaas": "test-iaas", }, } v, err := form.EncodeToValues(¶ms) c.Assert(err, check.IsNil) b := strings.NewReader(v.Encode()) req, err := http.NewRequest("POST", "/node", b) c.Assert(err, check.IsNil) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("Authorization", s.token.GetValue()) rec := httptest.NewRecorder() m := RunServer(true) m.ServeHTTP(rec, req) c.Assert(rec.Code, check.Equals, http.StatusCreated) c.Assert(rec.Body.String(), check.Equals, "") nodes, err := s.provisioner.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://test1.somewhere.com:2375") c.Assert(nodes[0].Metadata(), check.DeepEquals, map[string]string{ "id": "test1", "pool": "pool1", "iaas": "test-iaas", "iaas-id": "test1", }) c.Assert(eventtest.EventDesc{ Target: event.Target{Type: event.TargetTypeNode, Value: "http://test1.somewhere.com:2375"}, Owner: s.token.GetUserName(), Kind: "node.create", StartCustomData: []map[string]interface{}{ {"name": "Metadata.id", "value": "test1"}, {"name": "Metadata.pool", "value": "pool1"}, {"name": "Register", "value": ""}, }, }, eventtest.HasEvent) }
func (s *S) TestHealerHealNode(c *check.C) { factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1") iaas.RegisterIaasProvider("my-healer-iaas", factory) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaasInst.Addr = "addr2" config.Set("iaas:node-protocol", "http") config.Set("iaas:node-port", 2) defer config.Unset("iaas:node-protocol") defer config.Unset("iaas:node-port") p := provisiontest.ProvisionerInstance err = p.AddNode(provision.AddNodeOptions{ Address: "http://addr1:1", Metadata: map[string]string{"iaas": "my-healer-iaas"}, }) c.Assert(err, check.IsNil) healer := newNodeHealer(nodeHealerArgs{ FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Minute, }) healer.Shutdown() nodes, err := p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") machines, err := iaas.ListMachines() c.Assert(err, check.IsNil) c.Assert(machines, check.HasLen, 1) c.Assert(machines[0].Address, check.Equals, "addr1") created, err := healer.healNode(nodes[0]) c.Assert(err, check.IsNil) c.Assert(created.Address, check.Equals, "http://addr2:2") nodes, err = p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr2:2") machines, err = iaas.ListMachines() c.Assert(err, check.IsNil) c.Assert(machines, check.HasLen, 1) c.Assert(machines[0].Address, check.Equals, "addr2") }
func (s *HandlersSuite) TestRemoveNodeHandlerRemoveIaaS(c *check.C) { iaas.RegisterIaasProvider("my-xxx-iaas", newTestIaaS) machine, err := iaas.CreateMachineForIaaS("my-xxx-iaas", map[string]string{}) c.Assert(err, check.IsNil) mainDockerProvisioner.cluster, err = cluster.New(nil, &cluster.MapStorage{}) c.Assert(err, check.IsNil) _, err = mainDockerProvisioner.getCluster().Register(fmt.Sprintf("http://%s:2375", machine.Address), nil) c.Assert(err, check.IsNil) b := bytes.NewBufferString(fmt.Sprintf(`{"address": "http://%s:2375", "remove_iaas": "true"}`, machine.Address)) req, err := http.NewRequest("POST", "/node/remove", b) c.Assert(err, check.IsNil) rec := httptest.NewRecorder() err = removeNodeHandler(rec, req, nil) c.Assert(err, check.IsNil) nodes, err := mainDockerProvisioner.getCluster().Nodes() c.Assert(len(nodes), check.Equals, 0) _, err = iaas.FindMachineById(machine.Id) c.Assert(err, check.Equals, mgo.ErrNotFound) }
func (s *HandlersSuite) TestRemoveNodeHandlerWithoutRemoveIaaS(c *gocheck.C) { iaas.RegisterIaasProvider("some-iaas", TestIaaS{}) machine, err := iaas.CreateMachineForIaaS("some-iaas", map[string]string{}) c.Assert(err, gocheck.IsNil) dCluster, err = cluster.New(nil, &cluster.MapStorage{}) c.Assert(err, gocheck.IsNil) err = dCluster.Register(fmt.Sprintf("http://%s:4243", machine.Address), nil) c.Assert(err, gocheck.IsNil) b := bytes.NewBufferString(fmt.Sprintf(`{"address": "http://%s:4243", "remove_iaas": "false"}`, machine.Address)) req, err := http.NewRequest("POST", "/node/remove", b) c.Assert(err, gocheck.IsNil) rec := httptest.NewRecorder() err = removeNodeHandler(rec, req, nil) c.Assert(err, gocheck.IsNil) nodes, err := dCluster.Nodes() c.Assert(len(nodes), gocheck.Equals, 0) dbM, err := iaas.FindMachineById(machine.Id) c.Assert(err, gocheck.IsNil) c.Assert(dbM.Id, gocheck.Equals, machine.Id) }
func (s *S) TestMachinesDestroy(c *check.C) { iaas.RegisterIaasProvider("test-iaas", newTestIaaS) _, err := iaas.CreateMachineForIaaS("test-iaas", map[string]string{"id": "myid1"}) c.Assert(err, check.IsNil) recorder := httptest.NewRecorder() request, err := http.NewRequest("DELETE", "/iaas/machines/myid1", nil) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.token.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusOK) c.Assert(eventtest.EventDesc{ Target: event.Target{Type: event.TargetTypeIaas, Value: "test-iaas"}, Owner: s.token.GetUserName(), Kind: "machine.delete", StartCustomData: []map[string]interface{}{ {"name": ":machine_id", "value": "myid1"}, }, }, eventtest.HasEvent) }
func (s *S) TestTemplateUpdate(c *check.C) { iaas.RegisterIaasProvider("my-iaas", newTestIaaS) tpl1 := iaas.Template{ Name: "my-tpl", IaaSName: "my-iaas", Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "x", Value: "y"}, {Name: "a", Value: "b"}, }), } err := tpl1.Save() c.Assert(err, check.IsNil) tplParam := iaas.Template{ Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "x", Value: ""}, {Name: "y", Value: "8"}, {Name: "z", Value: "9"}, }), } bodyData, err := json.Marshal(tplParam) c.Assert(err, check.IsNil) body := bytes.NewBuffer(bodyData) recorder := httptest.NewRecorder() request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", body) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.token.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusOK) templates, err := iaas.ListTemplates() c.Assert(err, check.IsNil) c.Assert(templates, check.HasLen, 1) c.Assert(templates[0].Name, check.Equals, "my-tpl") c.Assert(templates[0].IaaSName, check.Equals, "my-iaas") sort.Sort(templates[0].Data) c.Assert(templates[0].Data, check.DeepEquals, iaas.TemplateDataList([]iaas.TemplateData{ {Name: "a", Value: "b"}, {Name: "y", Value: "8"}, {Name: "z", Value: "9"}, })) }
func (s *S) TestTemplateUpdateNotFound(c *check.C) { iaas.RegisterIaasProvider("my-iaas", newTestIaaS) tplParam := iaas.Template{ Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "x", Value: ""}, {Name: "y", Value: "8"}, {Name: "z", Value: "9"}, }), } v, err := form.EncodeToValues(&tplParam) c.Assert(err, check.IsNil) recorder := httptest.NewRecorder() request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", strings.NewReader(v.Encode())) c.Assert(err, check.IsNil) request.Header.Set("Content-Type", "application/x-www-form-urlencoded") request.Header.Set("Authorization", "bearer "+s.token.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusNotFound) c.Assert(recorder.Body.String(), check.Equals, "template not found\n") }
func (s *S) TestTemplateUpdateNotFound(c *check.C) { iaas.RegisterIaasProvider("my-iaas", newTestIaaS) tplParam := iaas.Template{ Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "x", Value: ""}, {Name: "y", Value: "8"}, {Name: "z", Value: "9"}, }), } bodyData, err := json.Marshal(tplParam) c.Assert(err, check.IsNil) body := bytes.NewBuffer(bodyData) recorder := httptest.NewRecorder() request, err := http.NewRequest("PUT", "/iaas/templates/my-tpl", body) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusNotFound) c.Assert(recorder.Body.String(), check.Equals, "template not found\n") }
func (s *S) TestTryHealingNodeDoubleCheck(c *check.C) { factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1") iaas.RegisterIaasProvider("my-healer-iaas", factory) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaasInst.Addr = "addr2" config.Set("iaas:node-protocol", "http") config.Set("iaas:node-port", 2) defer config.Unset("iaas:node-protocol") defer config.Unset("iaas:node-port") p := provisiontest.ProvisionerInstance err = p.AddNode(provision.AddNodeOptions{ Address: "http://addr1:1", Metadata: map[string]string{"iaas": "my-healer-iaas"}, }) c.Assert(err, check.IsNil) healer := newNodeHealer(nodeHealerArgs{ FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Minute, }) healer.started = time.Now().Add(-3 * time.Second) nodes, err := p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) healErr := healer.tryHealingNode(nodes[0], "something", nil) c.Assert(healErr, check.IsNil) nodes, err = p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") machines, err := iaas.ListMachines() c.Assert(err, check.IsNil) c.Assert(machines, check.HasLen, 1) c.Assert(machines[0].Address, check.Equals, "addr1") c.Assert(eventtest.EventDesc{ IsEmpty: true, }, eventtest.HasEvent) }
func (s *S) TestHealerHealNodeCreateMachineError(c *check.C) { factory, iaasInst := dockertest.NewHealerIaaSConstructorWithInst("127.0.0.1") iaas.RegisterIaasProvider("my-healer-iaas", factory) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaasInst.Addr = "localhost" iaasInst.Err = fmt.Errorf("my create machine error") node1, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) p, err := s.newFakeDockerProvisioner(node1.URL()) c.Assert(err, check.IsNil) defer p.Destroy() node1.PrepareFailure("pingErr", "/_ping") p.Cluster().StartActiveMonitoring(100 * time.Millisecond) time.Sleep(300 * time.Millisecond) p.Cluster().StopActiveMonitoring() healer := NewNodeHealer(NodeHealerArgs{ Provisioner: p, FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Minute, }) nodes, err := p.Cluster().UnfilteredNodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL())) c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1") c.Assert(nodes[0].FailureCount() > 0, check.Equals, true) nodes[0].Metadata["iaas"] = "my-healer-iaas" created, err := healer.healNode(&nodes[0]) c.Assert(err, check.ErrorMatches, ".*my create machine error.*") c.Assert(created.Address, check.Equals, "") c.Assert(nodes[0].FailureCount(), check.Equals, 0) nodes, err = p.Cluster().UnfilteredNodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(dockertest.URLPort(nodes[0].Address), check.Equals, dockertest.URLPort(node1.URL())) c.Assert(net.URLToHost(nodes[0].Address), check.Equals, "127.0.0.1") }
func (s *S) TestRemoveNodeHandlerWithRemoveIaaS(c *check.C) { iaas.RegisterIaasProvider("some-iaas", newTestIaaS) machine, err := iaas.CreateMachineForIaaS("some-iaas", map[string]string{"id": "m1"}) c.Assert(err, check.IsNil) err = s.provisioner.AddNode(provision.AddNodeOptions{ Address: fmt.Sprintf("http://%s:2375", machine.Address), }) c.Assert(err, check.IsNil) u := fmt.Sprintf("/node/http://%s:2375?remove-iaas=true", machine.Address) req, err := http.NewRequest("DELETE", u, nil) c.Assert(err, check.IsNil) req.Header.Set("Authorization", "bearer "+s.token.GetValue()) rec := httptest.NewRecorder() server := RunServer(true) server.ServeHTTP(rec, req) c.Assert(rec.Body.String(), check.Equals, "rebalancing...remove done!") c.Assert(rec.Code, check.Equals, http.StatusOK) nodes, err := s.provisioner.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 0) _, err = iaas.FindMachineById(machine.Id) c.Assert(err, check.Equals, mgo.ErrNotFound) }
func (s *S) TestHealerHealNodeCreateMachineError(c *check.C) { factory, iaasInst := iaasTesting.NewHealerIaaSConstructorWithInst("addr1") iaas.RegisterIaasProvider("my-healer-iaas", factory) _, err := iaas.CreateMachineForIaaS("my-healer-iaas", map[string]string{}) c.Assert(err, check.IsNil) iaasInst.Addr = "addr2" iaasInst.Err = fmt.Errorf("my create machine error") p := provisiontest.ProvisionerInstance err = p.AddNode(provision.AddNodeOptions{ Address: "http://addr1:1", Metadata: map[string]string{"iaas": "my-healer-iaas"}, }) c.Assert(err, check.IsNil) healer := newNodeHealer(nodeHealerArgs{ FailuresBeforeHealing: 1, WaitTimeNewMachine: time.Minute, }) healer.Shutdown() nodes, err := p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") fakeNode := nodes[0].(*provisiontest.FakeNode) fakeNode.SetHealth(1, false) c.Assert(fakeNode.FailureCount() > 0, check.Equals, true) created, err := healer.healNode(nodes[0]) c.Assert(err, check.ErrorMatches, ".*my create machine error.*") c.Assert(created, check.IsNil) c.Assert(fakeNode.FailureCount(), check.Equals, 0) nodes, err = p.ListNodes(nil) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 1) c.Assert(nodes[0].Address(), check.Equals, "http://addr1:1") c.Assert(nodes[0].Status(), check.Equals, "enabled") }
func (s *S) TestTemplateDestroy(c *check.C) { iaas.RegisterIaasProvider("ec2", newTestIaaS) tpl1 := iaas.Template{ Name: "tpl1", IaaSName: "ec2", Data: iaas.TemplateDataList([]iaas.TemplateData{ {Name: "key1", Value: "val1"}, {Name: "key2", Value: "val2"}, }), } err := tpl1.Save() c.Assert(err, check.IsNil) defer iaas.DestroyTemplate("tpl1") recorder := httptest.NewRecorder() request, err := http.NewRequest("DELETE", "/iaas/templates/tpl1", nil) c.Assert(err, check.IsNil) request.Header.Set("Authorization", "bearer "+s.admintoken.GetValue()) m := RunServer(true) m.ServeHTTP(recorder, request) c.Assert(recorder.Code, check.Equals, http.StatusOK) templates, err := iaas.ListTemplates() c.Assert(err, check.IsNil) c.Assert(templates, check.HasLen, 0) }
func init() { iaas.RegisterIaasProvider("digitalocean", newDigitalOceanIaas) }
func init() { iaas.RegisterIaasProvider("cloudstack", &CloudstackIaaS{}) }