func (s *S) TestRecreateBsContainersErrorInSomeContainers(c *check.C) { _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) servers := p.Servers() servers[0].PrepareFailure("failure-create", "/containers/create") defer servers[1].ResetFailure("failure-create") var buf safe.Buffer err = recreateContainers(p, &buf) c.Assert(err, check.ErrorMatches, `(?s)API error \(400\): failure-create.*failed to create container in .* \[.*\].*`) sort.Sort(cluster.NodeList(nodes)) client, err := nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 0) client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) }
func (s *S) TestEnsureContainersStartedForceStopOnlyOnFailure(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) var reqs []*http.Request server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) { reqs = append(reqs, r) }) c.Assert(err, check.IsNil) defer server.Stop() errCount := 0 server.CustomHandler("/containers/big-sibling", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reqs = append(reqs, r) if r.Method != "DELETE" { server.DefaultHandler().ServeHTTP(w, r) return } errCount++ if errCount > 2 { server.DefaultHandler().ServeHTTP(w, r) } else { w.WriteHeader(http.StatusInternalServerError) } })) p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) reqs = nil err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) var paths []string for _, r := range reqs { paths = append(paths, r.Method+" "+r.URL.Path) } c.Assert(paths, check.DeepEquals, []string{ "POST /images/create", "POST /containers/create", "POST /containers/big-sibling/stop", "DELETE /containers/big-sibling", "DELETE /containers/big-sibling", "DELETE /containers/big-sibling", "POST /containers/create", "GET /version", "POST /containers/big-sibling/start", }) c.Assert(reqs[3].URL.Query().Get("force"), check.Equals, "") c.Assert(reqs[4].URL.Query().Get("force"), check.Equals, "1") c.Assert(reqs[5].URL.Query().Get("force"), check.Equals, "1") containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) }
func (s *S) TestClusterHookBeforeCreateContainerIgnoresExistingError(c *check.C) { _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() var buf safe.Buffer err = recreateContainers(p, &buf) c.Assert(err, check.IsNil) nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) hook := ClusterHook{Provisioner: p} err = hook.RunClusterHook(cluster.HookEventBeforeContainerCreate, &nodes[0]) c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) client, err := nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err = client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) }
func (p *dockerProvisioner) Initialize() error { err := internalNodeContainer.RegisterQueueTask(p) if err != nil { return err } _, err = nodecontainer.InitializeBS() if err != nil { return errors.Wrap(err, "unable to initialize bs node container") } return p.initDockerCluster() }
func (s *S) TestEnsureContainersStartedGracefullyStop(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) var reqs []*http.Request server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) { reqs = append(reqs, r) }) c.Assert(err, check.IsNil) defer server.Stop() p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) var paths []string for _, r := range reqs { paths = append(paths, r.Method+" "+r.URL.Path) } c.Assert(paths, check.DeepEquals, []string{ "POST /images/create", "POST /containers/create", "GET /version", "POST /containers/big-sibling/start", }) reqs = nil err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) paths = nil for _, r := range reqs { paths = append(paths, r.Method+" "+r.URL.Path) } c.Assert(paths, check.DeepEquals, []string{ "POST /images/create", "POST /containers/create", "POST /containers/big-sibling/stop", "DELETE /containers/big-sibling", "POST /containers/create", "GET /version", "POST /containers/big-sibling/start", }) c.Assert(reqs[3].URL.Query().Get("force"), check.Equals, "") containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) }
func (s *S) TestEnsureContainersStartedAlreadyPinned(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) cont, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) c.Assert(err, check.IsNil) cont.PinnedImage = "myregistry/tsuru/bs@" + digest err = nodecontainer.AddNewContainer("", cont) c.Assert(err, check.IsNil) server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) server.DefaultHandler().ServeHTTP(w, r) w.Write([]byte(pullOutputDigest)) })) p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) err = client.PullImage(docker.PullImageOptions{ Repository: "base", }, docker.AuthConfiguration{}) c.Assert(err, check.IsNil) _, err = client.CreateContainer(docker.CreateContainerOptions{ Name: nodecontainer.BsDefaultName, Config: &docker.Config{Image: "base"}, HostConfig: &docker.HostConfig{}, }) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs@"+digest) c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart()) c.Assert(container.State.Running, check.Equals, true) nodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) c.Assert(err, check.IsNil) c.Assert(nodeContainer.PinnedImage, check.Equals, "myregistry/tsuru/bs@"+digest) }
func (s *S) TestRemoveNamedContainers(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) for i, n := range nodes { n.Metadata["pool"] = fmt.Sprintf("p-%d", i) _, err = p.Cluster().UpdateNode(n) c.Assert(err, check.IsNil) } server := p.Servers()[0] var paths []string server.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { paths = append(paths, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery) server.DefaultHandler().ServeHTTP(w, r) })) server2 := p.Servers()[1] var paths2 []string server2.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { paths2 = append(paths2, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery) server2.DefaultHandler().ServeHTTP(w, r) })) err = ensureContainersStarted(p, ioutil.Discard, true, nil) c.Assert(err, check.IsNil) paths = nil paths2 = nil expectedPaths := []string{"POST /containers/big-sibling/stop?t=10", "DELETE /containers/big-sibling?force=1"} err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "") c.Assert(err, check.IsNil) c.Assert(paths, check.DeepEquals, expectedPaths) c.Assert(paths2, check.DeepEquals, expectedPaths) err = ensureContainersStarted(p, ioutil.Discard, true, nil) c.Assert(err, check.IsNil) paths = nil paths2 = nil err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "p-1") c.Assert(err, check.IsNil) c.Assert(paths, check.DeepEquals, []string(nil)) c.Assert(paths2, check.DeepEquals, expectedPaths) }
func (s *S) TestRecreateBsContainers(c *check.C) { _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) defer p.Destroy() var buf safe.Buffer err = recreateContainers(p, &buf) c.Assert(err, check.IsNil) nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) c.Assert(nodes, check.HasLen, 2) client, err := nodes[0].Client() c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) client, err = nodes[1].Client() c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err = client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) // It runs in parallel, so we check both ordering output1 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s [] relaunching node container "big-sibling" in the node %s [] `, nodes[0].Address, nodes[1].Address) output2 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s [] relaunching node container "big-sibling" in the node %s [] `, nodes[1].Address, nodes[0].Address) if got := buf.String(); got != output1 && got != output2 { c.Errorf("Wrong output:\n%s", got) } }
func (s *S) TestEnsureContainersStartedTryCreatingAfterRmFailure(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/containers/big-sibling", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == "DELETE" { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("my error")) return } server.DefaultHandler().ServeHTTP(w, r) })) p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.ErrorMatches, `(?s).*API error \(500\): my error.*unable to remove old node-container.*container already exists.*unable to create new node-container.*`) }
func migrateBSEnvs() error { scheme, err := config.GetString("auth:scheme") if err != nil { scheme = nativeSchemeName } app.AuthScheme, err = auth.GetScheme(scheme) if err != nil { return err } _, err = nodecontainer.InitializeBS() if err != nil { return err } conn, err := db.Conn() if err != nil { return err } defer conn.Close() var entry map[string]interface{} err = conn.Collection("bsconfig").FindId("bs").One(&entry) if err != nil { if err == mgo.ErrNotFound { return nil } return err } image, _ := entry["image"].(string) envs, _ := entry["envs"].([]interface{}) var baseEnvs []string for _, envEntry := range envs { mapEntry, _ := envEntry.(map[string]interface{}) if mapEntry == nil { continue } name, _ := mapEntry["name"].(string) value, _ := mapEntry["value"].(string) baseEnvs = append(baseEnvs, fmt.Sprintf("%s=%s", name, value)) } bsNodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) if err != nil { return err } if len(baseEnvs) > 0 { bsNodeContainer.Config.Env = append(bsNodeContainer.Config.Env, baseEnvs...) } bsNodeContainer.PinnedImage = image err = nodecontainer.AddNewContainer("", bsNodeContainer) if err != nil { return err } pools, _ := entry["pools"].([]interface{}) for _, poolData := range pools { poolMap, _ := poolData.(map[string]interface{}) if poolMap == nil { continue } poolName, _ := poolMap["name"].(string) if poolName == "" { continue } envs, _ := poolMap["envs"].([]interface{}) var toAdd []string for _, envEntry := range envs { mapEntry, _ := envEntry.(map[string]interface{}) if mapEntry == nil { continue } name, _ := mapEntry["name"].(string) value, _ := mapEntry["value"].(string) toAdd = append(toAdd, fmt.Sprintf("%s=%s", name, value)) } if len(toAdd) > 0 { bsCont := nodecontainer.NodeContainerConfig{Name: nodecontainer.BsDefaultName} bsCont.Config.Env = append(bsCont.Config.Env, toAdd...) err = nodecontainer.AddNewContainer(poolName, &bsCont) if err != nil { return err } } } return nil }