func (s *S) TestRecreateBsContainersErrorInSomeContainers(c *check.C) {
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
servers := p.Servers()
servers[0].PrepareFailure("failure-create", "/containers/create")
defer servers[1].ResetFailure("failure-create")
var buf safe.Buffer
err = recreateContainers(p, &buf)
c.Assert(err, check.ErrorMatches, `(?s)API error \(400\): failure-create.*failed to create container in .* \[.*\].*`)
sort.Sort(cluster.NodeList(nodes))
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 0)
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
}
func (s *S) TestEnsureContainersStartedForceStopOnlyOnFailure(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
var reqs []*http.Request
server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
reqs = append(reqs, r)
})
c.Assert(err, check.IsNil)
defer server.Stop()
errCount := 0
server.CustomHandler("/containers/big-sibling", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reqs = append(reqs, r)
if r.Method != "DELETE" {
server.DefaultHandler().ServeHTTP(w, r)
return
}
errCount++
if errCount > 2 {
server.DefaultHandler().ServeHTTP(w, r)
} else {
w.WriteHeader(http.StatusInternalServerError)
}
}))
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
reqs = nil
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
var paths []string
for _, r := range reqs {
paths = append(paths, r.Method+" "+r.URL.Path)
}
c.Assert(paths, check.DeepEquals, []string{
"POST /images/create",
"POST /containers/create",
"POST /containers/big-sibling/stop",
"DELETE /containers/big-sibling",
"DELETE /containers/big-sibling",
"DELETE /containers/big-sibling",
"POST /containers/create",
"GET /version",
"POST /containers/big-sibling/start",
})
c.Assert(reqs[3].URL.Query().Get("force"), check.Equals, "")
c.Assert(reqs[4].URL.Query().Get("force"), check.Equals, "1")
c.Assert(reqs[5].URL.Query().Get("force"), check.Equals, "1")
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
}
func (s *S) TestClusterHookBeforeCreateContainerIgnoresExistingError(c *check.C) {
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
var buf safe.Buffer
err = recreateContainers(p, &buf)
c.Assert(err, check.IsNil)
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
hook := ClusterHook{Provisioner: p}
err = hook.RunClusterHook(cluster.HookEventBeforeContainerCreate, &nodes[0])
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err = client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
}
func (p *dockerProvisioner) Initialize() error {
err := internalNodeContainer.RegisterQueueTask(p)
if err != nil {
return err
}
_, err = nodecontainer.InitializeBS()
if err != nil {
return errors.Wrap(err, "unable to initialize bs node container")
}
return p.initDockerCluster()
}
func (s *S) TestEnsureContainersStartedGracefullyStop(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
var reqs []*http.Request
server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
reqs = append(reqs, r)
})
c.Assert(err, check.IsNil)
defer server.Stop()
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
var paths []string
for _, r := range reqs {
paths = append(paths, r.Method+" "+r.URL.Path)
}
c.Assert(paths, check.DeepEquals, []string{
"POST /images/create",
"POST /containers/create",
"GET /version",
"POST /containers/big-sibling/start",
})
reqs = nil
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
paths = nil
for _, r := range reqs {
paths = append(paths, r.Method+" "+r.URL.Path)
}
c.Assert(paths, check.DeepEquals, []string{
"POST /images/create",
"POST /containers/create",
"POST /containers/big-sibling/stop",
"DELETE /containers/big-sibling",
"POST /containers/create",
"GET /version",
"POST /containers/big-sibling/start",
})
c.Assert(reqs[3].URL.Query().Get("force"), check.Equals, "")
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
}
func (s *S) TestEnsureContainersStartedAlreadyPinned(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
cont, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
c.Assert(err, check.IsNil)
cont.PinnedImage = "myregistry/tsuru/bs@" + digest
err = nodecontainer.AddNewContainer("", cont)
c.Assert(err, check.IsNil)
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
server.DefaultHandler().ServeHTTP(w, r)
w.Write([]byte(pullOutputDigest))
}))
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
err = client.PullImage(docker.PullImageOptions{
Repository: "base",
}, docker.AuthConfiguration{})
c.Assert(err, check.IsNil)
_, err = client.CreateContainer(docker.CreateContainerOptions{
Name: nodecontainer.BsDefaultName,
Config: &docker.Config{Image: "base"},
HostConfig: &docker.HostConfig{},
})
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs@"+digest)
c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart())
c.Assert(container.State.Running, check.Equals, true)
nodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
c.Assert(err, check.IsNil)
c.Assert(nodeContainer.PinnedImage, check.Equals, "myregistry/tsuru/bs@"+digest)
}
func (s *S) TestRemoveNamedContainers(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
for i, n := range nodes {
n.Metadata["pool"] = fmt.Sprintf("p-%d", i)
_, err = p.Cluster().UpdateNode(n)
c.Assert(err, check.IsNil)
}
server := p.Servers()[0]
var paths []string
server.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
paths = append(paths, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery)
server.DefaultHandler().ServeHTTP(w, r)
}))
server2 := p.Servers()[1]
var paths2 []string
server2.CustomHandler("/containers/.*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
paths2 = append(paths2, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery)
server2.DefaultHandler().ServeHTTP(w, r)
}))
err = ensureContainersStarted(p, ioutil.Discard, true, nil)
c.Assert(err, check.IsNil)
paths = nil
paths2 = nil
expectedPaths := []string{"POST /containers/big-sibling/stop?t=10", "DELETE /containers/big-sibling?force=1"}
err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "")
c.Assert(err, check.IsNil)
c.Assert(paths, check.DeepEquals, expectedPaths)
c.Assert(paths2, check.DeepEquals, expectedPaths)
err = ensureContainersStarted(p, ioutil.Discard, true, nil)
c.Assert(err, check.IsNil)
paths = nil
paths2 = nil
err = RemoveNamedContainers(p, ioutil.Discard, "big-sibling", "p-1")
c.Assert(err, check.IsNil)
c.Assert(paths, check.DeepEquals, []string(nil))
c.Assert(paths2, check.DeepEquals, expectedPaths)
}
func (s *S) TestRecreateBsContainers(c *check.C) {
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
defer p.Destroy()
var buf safe.Buffer
err = recreateContainers(p, &buf)
c.Assert(err, check.IsNil)
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
c.Assert(nodes, check.HasLen, 2)
client, err := nodes[0].Client()
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
client, err = nodes[1].Client()
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err = client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
// It runs in parallel, so we check both ordering
output1 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s []
relaunching node container "big-sibling" in the node %s []
`, nodes[0].Address, nodes[1].Address)
output2 := fmt.Sprintf(`relaunching node container "big-sibling" in the node %s []
relaunching node container "big-sibling" in the node %s []
`, nodes[1].Address, nodes[0].Address)
if got := buf.String(); got != output1 && got != output2 {
c.Errorf("Wrong output:\n%s", got)
}
}
func (s *S) TestEnsureContainersStartedTryCreatingAfterRmFailure(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/containers/big-sibling", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "DELETE" {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("my error"))
return
}
server.DefaultHandler().ServeHTTP(w, r)
}))
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.ErrorMatches, `(?s).*API error \(500\): my error.*unable to remove old node-container.*container already exists.*unable to create new node-container.*`)
}
func migrateBSEnvs() error {
scheme, err := config.GetString("auth:scheme")
if err != nil {
scheme = nativeSchemeName
}
app.AuthScheme, err = auth.GetScheme(scheme)
if err != nil {
return err
}
_, err = nodecontainer.InitializeBS()
if err != nil {
return err
}
conn, err := db.Conn()
if err != nil {
return err
}
defer conn.Close()
var entry map[string]interface{}
err = conn.Collection("bsconfig").FindId("bs").One(&entry)
if err != nil {
if err == mgo.ErrNotFound {
return nil
}
return err
}
image, _ := entry["image"].(string)
envs, _ := entry["envs"].([]interface{})
var baseEnvs []string
for _, envEntry := range envs {
mapEntry, _ := envEntry.(map[string]interface{})
if mapEntry == nil {
continue
}
name, _ := mapEntry["name"].(string)
value, _ := mapEntry["value"].(string)
baseEnvs = append(baseEnvs, fmt.Sprintf("%s=%s", name, value))
}
bsNodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
if err != nil {
return err
}
if len(baseEnvs) > 0 {
bsNodeContainer.Config.Env = append(bsNodeContainer.Config.Env, baseEnvs...)
}
bsNodeContainer.PinnedImage = image
err = nodecontainer.AddNewContainer("", bsNodeContainer)
if err != nil {
return err
}
pools, _ := entry["pools"].([]interface{})
for _, poolData := range pools {
poolMap, _ := poolData.(map[string]interface{})
if poolMap == nil {
continue
}
poolName, _ := poolMap["name"].(string)
if poolName == "" {
continue
}
envs, _ := poolMap["envs"].([]interface{})
var toAdd []string
for _, envEntry := range envs {
mapEntry, _ := envEntry.(map[string]interface{})
if mapEntry == nil {
continue
}
name, _ := mapEntry["name"].(string)
value, _ := mapEntry["value"].(string)
toAdd = append(toAdd, fmt.Sprintf("%s=%s", name, value))
}
if len(toAdd) > 0 {
bsCont := nodecontainer.NodeContainerConfig{Name: nodecontainer.BsDefaultName}
bsCont.Config.Env = append(bsCont.Config.Env, toAdd...)
err = nodecontainer.AddNewContainer(poolName, &bsCont)
if err != nil {
return err
}
}
}
return nil
}