func (s *S) TestRunContainerHealerMaxCounterExceeded(c *check.C) {
conts := []container.Container{
{ID: "cont1"}, {ID: "cont2"}, {ID: "cont3"}, {ID: "cont4"},
{ID: "cont5"}, {ID: "cont6"}, {ID: "cont7"}, {ID: "cont8"},
}
for i := 0; i < len(conts)-1; i++ {
evt, err := NewHealingEvent(conts[i])
c.Assert(err, check.IsNil)
err = evt.Update(conts[i+1], nil)
c.Assert(err, check.IsNil)
}
toMoveCont := conts[7]
toMoveCont.LastSuccessStatusUpdate = time.Now().Add(-2 * time.Minute)
p, err := dockertest.NewFakeDockerProvisioner()
c.Assert(err, check.IsNil)
defer p.Destroy()
healer := NewContainerHealer(ContainerHealerArgs{Provisioner: p, Locker: dockertest.NewFakeLocker()})
err = healer.healContainerIfNeeded(toMoveCont)
c.Assert(err, check.ErrorMatches, "Containers healing: number of healings for container cont8 in the last 30 minutes exceeds limit of 3: 7")
healingColl, err := healingCollection()
c.Assert(err, check.IsNil)
defer healingColl.Close()
var events []HealingEvent
err = healingColl.Find(nil).All(&events)
c.Assert(err, check.IsNil)
c.Assert(events, check.HasLen, 7)
}
func (s *S) TestEnsureContainersStartedForceStopOnlyOnFailure(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
var reqs []*http.Request
server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
reqs = append(reqs, r)
})
c.Assert(err, check.IsNil)
defer server.Stop()
errCount := 0
server.CustomHandler("/containers/big-sibling", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reqs = append(reqs, r)
if r.Method != "DELETE" {
server.DefaultHandler().ServeHTTP(w, r)
return
}
errCount++
if errCount > 2 {
server.DefaultHandler().ServeHTTP(w, r)
} else {
w.WriteHeader(http.StatusInternalServerError)
}
}))
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
reqs = nil
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
var paths []string
for _, r := range reqs {
paths = append(paths, r.Method+" "+r.URL.Path)
}
c.Assert(paths, check.DeepEquals, []string{
"POST /images/create",
"POST /containers/create",
"POST /containers/big-sibling/stop",
"DELETE /containers/big-sibling",
"DELETE /containers/big-sibling",
"DELETE /containers/big-sibling",
"POST /containers/create",
"GET /version",
"POST /containers/big-sibling/start",
})
c.Assert(reqs[3].URL.Query().Get("force"), check.Equals, "")
c.Assert(reqs[4].URL.Query().Get("force"), check.Equals, "1")
c.Assert(reqs[5].URL.Query().Get("force"), check.Equals, "1")
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
}
func (s *S) TestCreateBsContainerSocketAndCustomSysLogPort(c *check.C) {
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
server.DefaultHandler().ServeHTTP(w, r)
w.Write([]byte(pullOutputNoDigest))
}))
config.Set("host", "127.0.0.1:8080")
config.Set("docker:bs:image", "myregistry/tsuru/bs")
config.Set("docker:bs:socket", "/tmp/docker.sock")
defer config.Unset("docker:bs:socket")
config.Set("docker:bs:syslog-port", 1519)
defer config.Unset("docker:bs:syslog-port")
conf, err := LoadConfig(nil)
c.Assert(err, check.IsNil)
conf.Add("VAR1", "VALUE1")
conf.Add("VAR2", "VALUE2")
conf.Add("TSURU_ENDPOINT", "ignored")
conf.AddPool("pool1", "VAR2", "VALUE_FOR_POOL1")
conf.AddPool("pool2", "VAR2", "VALUE_FOR_POOL2")
conf.AddPool("pool1", "SYSLOG_LISTEN_ADDRESS", "alsoignored")
err = conf.SaveEnvs()
c.Assert(err, check.IsNil)
p, err := dockertest.NewFakeDockerProvisioner()
c.Assert(err, check.IsNil)
defer p.Destroy()
err = createContainer(server.URL(), "pool1", p, true)
c.Assert(err, check.IsNil)
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
c.Assert(container.HostConfig.Binds, check.DeepEquals, []string{"/tmp/docker.sock:/var/run/docker.sock:rw"})
c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs")
c.Assert(container.State.Running, check.Equals, true)
conf, err = LoadConfig(nil)
c.Assert(err, check.IsNil)
token, _ := getToken(conf)
expectedEnv := []string{
"DOCKER_ENDPOINT=unix:///var/run/docker.sock",
"TSURU_ENDPOINT=http://127.0.0.1:8080/",
"TSURU_TOKEN=" + token,
"SYSLOG_LISTEN_ADDRESS=udp://0.0.0.0:1519",
"VAR1=VALUE1",
"VAR2=VALUE_FOR_POOL1",
}
sort.Strings(expectedEnv)
sort.Strings(container.Config.Env)
c.Assert(container.Config.Env, check.DeepEquals, expectedEnv)
c.Assert(conf.GetExtraString("image"), check.Equals, "myregistry/tsuru/bs")
}
func (s *S) newFakeDockerProvisioner(servers ...string) (*dockertest.FakeDockerProvisioner, error) {
p, err := dockertest.NewFakeDockerProvisioner(servers...)
if err != nil {
return nil, err
}
p.SetContainers("127.0.0.1", nil)
p.SetContainers("localhost", nil)
err = bs.RegisterQueueTask(p)
return p, err
}
func (s *S) TestCreateBsContainerTaggedBs(c *check.C) {
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
server.DefaultHandler().ServeHTTP(w, r)
w.Write([]byte(pullOutputDigest))
}))
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
err = client.PullImage(docker.PullImageOptions{
Repository: "base",
}, docker.AuthConfiguration{})
_, err = client.CreateContainer(docker.CreateContainerOptions{
Name: "big-sibling",
Config: &docker.Config{Image: "base"},
HostConfig: &docker.HostConfig{},
})
c.Assert(err, check.IsNil)
config.Set("host", "127.0.0.1:8080")
config.Set("docker:bs:image", "localhost:5000/myregistry/tsuru/bs:v1")
p, err := dockertest.NewFakeDockerProvisioner()
c.Assert(err, check.IsNil)
defer p.Destroy()
err = createContainer(server.URL(), "pool1", p, true)
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
c.Assert(container.Config.Image, check.Equals, "localhost:5000/myregistry/tsuru/bs:v1")
c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart())
c.Assert(container.HostConfig.Privileged, check.Equals, true)
c.Assert(container.HostConfig.NetworkMode, check.Equals, "host")
c.Assert(container.State.Running, check.Equals, true)
expectedEnv := map[string]string{
"DOCKER_ENDPOINT": server.URL(),
"TSURU_ENDPOINT": "http://127.0.0.1:8080/",
"TSURU_TOKEN": "abc123",
"SYSLOG_LISTEN_ADDRESS": "udp://0.0.0.0:1514",
}
gotEnv := parseEnvs(container.Config.Env)
_, ok := gotEnv["TSURU_TOKEN"]
c.Assert(ok, check.Equals, true)
gotEnv["TSURU_TOKEN"] = expectedEnv["TSURU_TOKEN"]
c.Assert(gotEnv, check.DeepEquals, expectedEnv)
conf, err := LoadConfig()
c.Assert(err, check.IsNil)
c.Assert(conf.Image, check.Equals, "localhost:5000/myregistry/tsuru/bs:v1")
}
func (s *S) TestEnsureContainersStartedGracefullyStop(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
var reqs []*http.Request
server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
reqs = append(reqs, r)
})
c.Assert(err, check.IsNil)
defer server.Stop()
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
var paths []string
for _, r := range reqs {
paths = append(paths, r.Method+" "+r.URL.Path)
}
c.Assert(paths, check.DeepEquals, []string{
"POST /images/create",
"POST /containers/create",
"GET /version",
"POST /containers/big-sibling/start",
})
reqs = nil
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
paths = nil
for _, r := range reqs {
paths = append(paths, r.Method+" "+r.URL.Path)
}
c.Assert(paths, check.DeepEquals, []string{
"POST /images/create",
"POST /containers/create",
"POST /containers/big-sibling/stop",
"DELETE /containers/big-sibling",
"POST /containers/create",
"GET /version",
"POST /containers/big-sibling/start",
})
c.Assert(reqs[3].URL.Query().Get("force"), check.Equals, "")
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
}
func (s *S) TestCreateBsContainer(c *check.C) {
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
server.DefaultHandler().ServeHTTP(w, r)
w.Write([]byte(pullOutputDigest))
}))
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
err = client.PullImage(docker.PullImageOptions{
Repository: "base",
}, docker.AuthConfiguration{})
_, err = client.CreateContainer(docker.CreateContainerOptions{
Name: "big-sibling",
Config: &docker.Config{Image: "base"},
HostConfig: &docker.HostConfig{},
})
c.Assert(err, check.IsNil)
config.Set("host", "127.0.0.1:8080")
config.Set("docker:bs:image", "myregistry/tsuru/bs")
p, err := dockertest.NewFakeDockerProvisioner()
c.Assert(err, check.IsNil)
defer p.Destroy()
err = createContainer(server.URL(), "pool1", p, true)
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs")
c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart())
c.Assert(container.State.Running, check.Equals, true)
conf, err := LoadConfig(nil)
c.Assert(err, check.IsNil)
token, _ := getToken(conf)
expectedEnv := []string{
"DOCKER_ENDPOINT=" + server.URL(),
"TSURU_ENDPOINT=http://127.0.0.1:8080/",
"TSURU_TOKEN=" + token,
"SYSLOG_LISTEN_ADDRESS=udp://0.0.0.0:1514",
}
sort.Strings(expectedEnv)
sort.Strings(container.Config.Env)
c.Assert(container.Config.Env, check.DeepEquals, expectedEnv)
c.Assert(conf.GetExtraString("image"), check.Equals, "myregistry/tsuru/bs@"+digest)
}
func (s *S) TestEnsureContainersStartedAlreadyPinned(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
cont, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
c.Assert(err, check.IsNil)
cont.PinnedImage = "myregistry/tsuru/bs@" + digest
err = nodecontainer.AddNewContainer("", cont)
c.Assert(err, check.IsNil)
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
server.DefaultHandler().ServeHTTP(w, r)
w.Write([]byte(pullOutputDigest))
}))
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
err = client.PullImage(docker.PullImageOptions{
Repository: "base",
}, docker.AuthConfiguration{})
c.Assert(err, check.IsNil)
_, err = client.CreateContainer(docker.CreateContainerOptions{
Name: nodecontainer.BsDefaultName,
Config: &docker.Config{Image: "base"},
HostConfig: &docker.HostConfig{},
})
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs@"+digest)
c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart())
c.Assert(container.State.Running, check.Equals, true)
nodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
c.Assert(err, check.IsNil)
c.Assert(nodeContainer.PinnedImage, check.Equals, "myregistry/tsuru/bs@"+digest)
}
func (s *S) TestEnsureContainersStartedPinImgInChild(c *check.C) {
err := nodecontainer.AddNewContainer("", &nodecontainer.NodeContainerConfig{
Name: "c1",
Config: docker.Config{
Image: "myrootimg",
},
})
c.Assert(err, check.IsNil)
err = nodecontainer.AddNewContainer("p1", &nodecontainer.NodeContainerConfig{
Name: "c1",
Config: docker.Config{
Image: "myregistry/tsuru/bs",
},
})
c.Assert(err, check.IsNil)
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
server.DefaultHandler().ServeHTTP(w, r)
w.Write([]byte(pullOutputDigest))
}))
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
node, err := p.Cluster().GetNode(server.URL())
c.Assert(err, check.IsNil)
node.Metadata["pool"] = "p1"
_, err = p.Cluster().UpdateNode(node)
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
all, err := nodecontainer.LoadNodeContainersForPoolsMerge("c1", false)
c.Assert(err, check.IsNil)
c.Assert(all, check.DeepEquals, map[string]nodecontainer.NodeContainerConfig{
"": {Name: "c1", PinnedImage: "", Config: docker.Config{Image: "myrootimg"}},
"p1": {Name: "c1", PinnedImage: "myregistry/tsuru/bs@" + digest, Config: docker.Config{Image: "myregistry/tsuru/bs"}},
})
}
func (s *S) TestEnsureContainersStartedTryCreatingAfterRmFailure(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/containers/big-sibling", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "DELETE" {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("my error"))
return
}
server.DefaultHandler().ServeHTTP(w, r)
}))
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.ErrorMatches, `(?s).*API error \(500\): my error.*unable to remove old node-container.*container already exists.*unable to create new node-container.*`)
}
func (s *S) TestCreateBsContainerSocketAndCustomSysLogPort(c *check.C) {
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
server.DefaultHandler().ServeHTTP(w, r)
w.Write([]byte(pullOutputNoDigest))
}))
config.Set("host", "127.0.0.1:8080")
config.Set("docker:bs:image", "myregistry/tsuru/bs")
config.Set("docker:bs:socket", "/tmp/docker.sock")
defer config.Unset("docker:bs:socket")
config.Set("docker:bs:syslog-port", 1519)
defer config.Unset("docker:bs:syslog-port")
err = SaveEnvs(EnvMap{
"VAR1": "VALUE1",
"VAR2": "VALUE2",
}, PoolEnvMap{
"pool1": EnvMap{
"VAR2": "VALUE_FOR_POOL1",
},
"pool2": EnvMap{
"VAR2": "VALUE_FOR_POOL2",
},
})
c.Assert(err, check.IsNil)
p, err := dockertest.NewFakeDockerProvisioner()
c.Assert(err, check.IsNil)
defer p.Destroy()
err = createContainer(server.URL(), "pool1", p, true)
c.Assert(err, check.IsNil)
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, "big-sibling")
c.Assert(container.HostConfig.Binds, check.DeepEquals, []string{"/tmp/docker.sock:/var/run/docker.sock:rw"})
c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs")
c.Assert(container.State.Running, check.Equals, true)
expectedEnv := map[string]string{
"DOCKER_ENDPOINT": "unix:///var/run/docker.sock",
"TSURU_ENDPOINT": "http://127.0.0.1:8080/",
"TSURU_TOKEN": "abc123",
"SYSLOG_LISTEN_ADDRESS": "udp://0.0.0.0:1519",
"VAR1": "VALUE1",
"VAR2": "VALUE_FOR_POOL1",
}
gotEnv := parseEnvs(container.Config.Env)
_, ok := gotEnv["TSURU_TOKEN"]
c.Assert(ok, check.Equals, true)
gotEnv["TSURU_TOKEN"] = expectedEnv["TSURU_TOKEN"]
c.Assert(gotEnv, check.DeepEquals, expectedEnv)
conf, err := LoadConfig()
c.Assert(err, check.IsNil)
c.Assert(conf.Image, check.Equals, "myregistry/tsuru/bs")
}