func (s *S) TestRunContainerHealerMaxCounterExceeded(c *check.C) { conts := []container.Container{ {ID: "cont1"}, {ID: "cont2"}, {ID: "cont3"}, {ID: "cont4"}, {ID: "cont5"}, {ID: "cont6"}, {ID: "cont7"}, {ID: "cont8"}, } for i := 0; i < len(conts)-1; i++ { evt, err := NewHealingEvent(conts[i]) c.Assert(err, check.IsNil) err = evt.Update(conts[i+1], nil) c.Assert(err, check.IsNil) } toMoveCont := conts[7] toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute) p, err := dockertest.NewFakeDockerProvisioner() c.Assert(err, check.IsNil) defer p.Destroy() healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()}) err = healer.healContainerIfNeeded(toMoveCont) c.Assert(err, check.ErrorMatches, "Containers healing: number of healings for container cont8 in the last 30 minutes exceeds limit of 3: 7") healingColl, err := healingCollection() c.Assert(err, check.IsNil) defer healingColl.Close() var events []HealingEvent err = healingColl.Find(nil).All(&events) c.Assert(err, check.IsNil) c.Assert(events, check.HasLen, 7) }
func (s *S) TestEnsureContainersStartedForceStopOnlyOnFailure(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) var reqs []*http.Request server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) { reqs = append(reqs, r) }) c.Assert(err, check.IsNil) defer server.Stop() errCount := 0 server.CustomHandler("/containers/big-sibling", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reqs = append(reqs, r) if r.Method != "DELETE" { server.DefaultHandler().ServeHTTP(w, r) return } errCount++ if errCount > 2 { server.DefaultHandler().ServeHTTP(w, r) } else { w.WriteHeader(http.StatusInternalServerError) } })) p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) reqs = nil err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) var paths []string for _, r := range reqs { paths = append(paths, r.Method+" "+r.URL.Path) } c.Assert(paths, check.DeepEquals, []string{ "POST /images/create", "POST /containers/create", "POST /containers/big-sibling/stop", "DELETE /containers/big-sibling", "DELETE /containers/big-sibling", "DELETE /containers/big-sibling", "POST /containers/create", "GET /version", "POST /containers/big-sibling/start", }) c.Assert(reqs[3].URL.Query().Get("force"), check.Equals, "") c.Assert(reqs[4].URL.Query().Get("force"), check.Equals, "1") c.Assert(reqs[5].URL.Query().Get("force"), check.Equals, "1") containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) }
func (s *S) TestCreateBsContainerSocketAndCustomSysLogPort(c *check.C) { server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) server.DefaultHandler().ServeHTTP(w, r) w.Write([]byte(pullOutputNoDigest)) })) config.Set("host", "127.0.0.1:8080") config.Set("docker:bs:image", "myregistry/tsuru/bs") config.Set("docker:bs:socket", "/tmp/docker.sock") defer config.Unset("docker:bs:socket") config.Set("docker:bs:syslog-port", 1519) defer config.Unset("docker:bs:syslog-port") conf, err := LoadConfig(nil) c.Assert(err, check.IsNil) conf.Add("VAR1", "VALUE1") conf.Add("VAR2", "VALUE2") conf.Add("TSURU_ENDPOINT", "ignored") conf.AddPool("pool1", "VAR2", "VALUE_FOR_POOL1") conf.AddPool("pool2", "VAR2", "VALUE_FOR_POOL2") conf.AddPool("pool1", "SYSLOG_LISTEN_ADDRESS", "alsoignored") err = conf.SaveEnvs() c.Assert(err, check.IsNil) p, err := dockertest.NewFakeDockerProvisioner() c.Assert(err, check.IsNil) defer p.Destroy() err = createContainer(server.URL(), "pool1", p, true) c.Assert(err, check.IsNil) client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") c.Assert(container.HostConfig.Binds, check.DeepEquals, []string{"/tmp/docker.sock:/var/run/docker.sock:rw"}) c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs") c.Assert(container.State.Running, check.Equals, true) conf, err = LoadConfig(nil) c.Assert(err, check.IsNil) token, _ := getToken(conf) expectedEnv := []string{ "DOCKER_ENDPOINT=unix:///var/run/docker.sock", "TSURU_ENDPOINT=http://127.0.0.1:8080/", "TSURU_TOKEN=" + token, "SYSLOG_LISTEN_ADDRESS=udp://0.0.0.0:1519", "VAR1=VALUE1", "VAR2=VALUE_FOR_POOL1", } sort.Strings(expectedEnv) sort.Strings(container.Config.Env) c.Assert(container.Config.Env, check.DeepEquals, expectedEnv) c.Assert(conf.GetExtraString("image"), check.Equals, "myregistry/tsuru/bs") }
func (s *S) newFakeDockerProvisioner(servers ...string) (*dockertest.FakeDockerProvisioner, error) { p, err := dockertest.NewFakeDockerProvisioner(servers...) if err != nil { return nil, err } p.SetContainers("127.0.0.1", nil) p.SetContainers("localhost", nil) err = bs.RegisterQueueTask(p) return p, err }
func (s *S) TestCreateBsContainerTaggedBs(c *check.C) { server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) server.DefaultHandler().ServeHTTP(w, r) w.Write([]byte(pullOutputDigest)) })) client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) err = client.PullImage(docker.PullImageOptions{ Repository: "base", }, docker.AuthConfiguration{}) _, err = client.CreateContainer(docker.CreateContainerOptions{ Name: "big-sibling", Config: &docker.Config{Image: "base"}, HostConfig: &docker.HostConfig{}, }) c.Assert(err, check.IsNil) config.Set("host", "127.0.0.1:8080") config.Set("docker:bs:image", "localhost:5000/myregistry/tsuru/bs:v1") p, err := dockertest.NewFakeDockerProvisioner() c.Assert(err, check.IsNil) defer p.Destroy() err = createContainer(server.URL(), "pool1", p, true) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") c.Assert(container.Config.Image, check.Equals, "localhost:5000/myregistry/tsuru/bs:v1") c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart()) c.Assert(container.HostConfig.Privileged, check.Equals, true) c.Assert(container.HostConfig.NetworkMode, check.Equals, "host") c.Assert(container.State.Running, check.Equals, true) expectedEnv := map[string]string{ "DOCKER_ENDPOINT": server.URL(), "TSURU_ENDPOINT": "http://127.0.0.1:8080/", "TSURU_TOKEN": "abc123", "SYSLOG_LISTEN_ADDRESS": "udp://0.0.0.0:1514", } gotEnv := parseEnvs(container.Config.Env) _, ok := gotEnv["TSURU_TOKEN"] c.Assert(ok, check.Equals, true) gotEnv["TSURU_TOKEN"] = expectedEnv["TSURU_TOKEN"] c.Assert(gotEnv, check.DeepEquals, expectedEnv) conf, err := LoadConfig() c.Assert(err, check.IsNil) c.Assert(conf.Image, check.Equals, "localhost:5000/myregistry/tsuru/bs:v1") }
func (s *S) TestEnsureContainersStartedGracefullyStop(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) var reqs []*http.Request server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) { reqs = append(reqs, r) }) c.Assert(err, check.IsNil) defer server.Stop() p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) var paths []string for _, r := range reqs { paths = append(paths, r.Method+" "+r.URL.Path) } c.Assert(paths, check.DeepEquals, []string{ "POST /images/create", "POST /containers/create", "GET /version", "POST /containers/big-sibling/start", }) reqs = nil err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) paths = nil for _, r := range reqs { paths = append(paths, r.Method+" "+r.URL.Path) } c.Assert(paths, check.DeepEquals, []string{ "POST /images/create", "POST /containers/create", "POST /containers/big-sibling/stop", "DELETE /containers/big-sibling", "POST /containers/create", "GET /version", "POST /containers/big-sibling/start", }) c.Assert(reqs[3].URL.Query().Get("force"), check.Equals, "") containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) }
func (s *S) TestCreateBsContainer(c *check.C) { server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) server.DefaultHandler().ServeHTTP(w, r) w.Write([]byte(pullOutputDigest)) })) client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) err = client.PullImage(docker.PullImageOptions{ Repository: "base", }, docker.AuthConfiguration{}) _, err = client.CreateContainer(docker.CreateContainerOptions{ Name: "big-sibling", Config: &docker.Config{Image: "base"}, HostConfig: &docker.HostConfig{}, }) c.Assert(err, check.IsNil) config.Set("host", "127.0.0.1:8080") config.Set("docker:bs:image", "myregistry/tsuru/bs") p, err := dockertest.NewFakeDockerProvisioner() c.Assert(err, check.IsNil) defer p.Destroy() err = createContainer(server.URL(), "pool1", p, true) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs") c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart()) c.Assert(container.State.Running, check.Equals, true) conf, err := LoadConfig(nil) c.Assert(err, check.IsNil) token, _ := getToken(conf) expectedEnv := []string{ "DOCKER_ENDPOINT=" + server.URL(), "TSURU_ENDPOINT=http://127.0.0.1:8080/", "TSURU_TOKEN=" + token, "SYSLOG_LISTEN_ADDRESS=udp://0.0.0.0:1514", } sort.Strings(expectedEnv) sort.Strings(container.Config.Env) c.Assert(container.Config.Env, check.DeepEquals, expectedEnv) c.Assert(conf.GetExtraString("image"), check.Equals, "myregistry/tsuru/bs@"+digest) }
func (s *S) TestEnsureContainersStartedAlreadyPinned(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) cont, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) c.Assert(err, check.IsNil) cont.PinnedImage = "myregistry/tsuru/bs@" + digest err = nodecontainer.AddNewContainer("", cont) c.Assert(err, check.IsNil) server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) server.DefaultHandler().ServeHTTP(w, r) w.Write([]byte(pullOutputDigest)) })) p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) err = client.PullImage(docker.PullImageOptions{ Repository: "base", }, docker.AuthConfiguration{}) c.Assert(err, check.IsNil) _, err = client.CreateContainer(docker.CreateContainerOptions{ Name: nodecontainer.BsDefaultName, Config: &docker.Config{Image: "base"}, HostConfig: &docker.HostConfig{}, }) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs@"+digest) c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart()) c.Assert(container.State.Running, check.Equals, true) nodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) c.Assert(err, check.IsNil) c.Assert(nodeContainer.PinnedImage, check.Equals, "myregistry/tsuru/bs@"+digest) }
func (s *S) TestEnsureContainersStartedPinImgInChild(c *check.C) { err := nodecontainer.AddNewContainer("", &nodecontainer.NodeContainerConfig{ Name: "c1", Config: docker.Config{ Image: "myrootimg", }, }) c.Assert(err, check.IsNil) err = nodecontainer.AddNewContainer("p1", &nodecontainer.NodeContainerConfig{ Name: "c1", Config: docker.Config{ Image: "myregistry/tsuru/bs", }, }) c.Assert(err, check.IsNil) server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) server.DefaultHandler().ServeHTTP(w, r) w.Write([]byte(pullOutputDigest)) })) p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() node, err := p.Cluster().GetNode(server.URL()) c.Assert(err, check.IsNil) node.Metadata["pool"] = "p1" _, err = p.Cluster().UpdateNode(node) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) all, err := nodecontainer.LoadNodeContainersForPoolsMerge("c1", false) c.Assert(err, check.IsNil) c.Assert(all, check.DeepEquals, map[string]nodecontainer.NodeContainerConfig{ "": {Name: "c1", PinnedImage: "", Config: docker.Config{Image: "myrootimg"}}, "p1": {Name: "c1", PinnedImage: "myregistry/tsuru/bs@" + digest, Config: docker.Config{Image: "myregistry/tsuru/bs"}}, }) }
func (s *S) TestEnsureContainersStartedTryCreatingAfterRmFailure(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/containers/big-sibling", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == "DELETE" { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("my error")) return } server.DefaultHandler().ServeHTTP(w, r) })) p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.ErrorMatches, `(?s).*API error \(500\): my error.*unable to remove old node-container.*container already exists.*unable to create new node-container.*`) }
func (s *S) TestCreateBsContainerSocketAndCustomSysLogPort(c *check.C) { server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) server.DefaultHandler().ServeHTTP(w, r) w.Write([]byte(pullOutputNoDigest)) })) config.Set("host", "127.0.0.1:8080") config.Set("docker:bs:image", "myregistry/tsuru/bs") config.Set("docker:bs:socket", "/tmp/docker.sock") defer config.Unset("docker:bs:socket") config.Set("docker:bs:syslog-port", 1519) defer config.Unset("docker:bs:syslog-port") err = SaveEnvs(EnvMap{ "VAR1": "VALUE1", "VAR2": "VALUE2", }, PoolEnvMap{ "pool1": EnvMap{ "VAR2": "VALUE_FOR_POOL1", }, "pool2": EnvMap{ "VAR2": "VALUE_FOR_POOL2", }, }) c.Assert(err, check.IsNil) p, err := dockertest.NewFakeDockerProvisioner() c.Assert(err, check.IsNil) defer p.Destroy() err = createContainer(server.URL(), "pool1", p, true) c.Assert(err, check.IsNil) client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, "big-sibling") c.Assert(container.HostConfig.Binds, check.DeepEquals, []string{"/tmp/docker.sock:/var/run/docker.sock:rw"}) c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs") c.Assert(container.State.Running, check.Equals, true) expectedEnv := map[string]string{ "DOCKER_ENDPOINT": "unix:///var/run/docker.sock", "TSURU_ENDPOINT": "http://127.0.0.1:8080/", "TSURU_TOKEN": "abc123", "SYSLOG_LISTEN_ADDRESS": "udp://0.0.0.0:1519", "VAR1": "VALUE1", "VAR2": "VALUE_FOR_POOL1", } gotEnv := parseEnvs(container.Config.Env) _, ok := gotEnv["TSURU_TOKEN"] c.Assert(ok, check.Equals, true) gotEnv["TSURU_TOKEN"] = expectedEnv["TSURU_TOKEN"] c.Assert(gotEnv, check.DeepEquals, expectedEnv) conf, err := LoadConfig() c.Assert(err, check.IsNil) c.Assert(conf.Image, check.Equals, "myregistry/tsuru/bs") }