func (s *S) TestEnsureContainersStartedAlreadyPinned(c *check.C) {
config.Set("docker:bs:image", "myregistry/tsuru/bs")
_, err := nodecontainer.InitializeBS()
c.Assert(err, check.IsNil)
cont, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
c.Assert(err, check.IsNil)
cont.PinnedImage = "myregistry/tsuru/bs@" + digest
err = nodecontainer.AddNewContainer("", cont)
c.Assert(err, check.IsNil)
server, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server.Stop()
server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
server.DefaultHandler().ServeHTTP(w, r)
w.Write([]byte(pullOutputDigest))
}))
p, err := dockertest.NewFakeDockerProvisioner(server.URL())
c.Assert(err, check.IsNil)
defer p.Destroy()
client, err := docker.NewClient(server.URL())
c.Assert(err, check.IsNil)
err = client.PullImage(docker.PullImageOptions{
Repository: "base",
}, docker.AuthConfiguration{})
c.Assert(err, check.IsNil)
_, err = client.CreateContainer(docker.CreateContainerOptions{
Name: nodecontainer.BsDefaultName,
Config: &docker.Config{Image: "base"},
HostConfig: &docker.HostConfig{},
})
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
container, err := client.InspectContainer(containers[0].ID)
c.Assert(err, check.IsNil)
c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName)
c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs@"+digest)
c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart())
c.Assert(container.State.Running, check.Equals, true)
nodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
c.Assert(err, check.IsNil)
c.Assert(nodeContainer.PinnedImage, check.Equals, "myregistry/tsuru/bs@"+digest)
}
func ensureContainersStarted(p DockerProvisioner, w io.Writer, relaunch bool, names []string, nodes ...cluster.Node) error {
if w == nil {
w = ioutil.Discard
}
var err error
if len(names) == 0 {
names, err = nodecontainer.AllNodeContainersNames()
if err != nil {
return err
}
}
if len(nodes) == 0 {
nodes, err = p.Cluster().UnfilteredNodes()
if err != nil {
return err
}
}
errChan := make(chan error, len(nodes)*len(names))
wg := sync.WaitGroup{}
log.Debugf("[node containers] recreating %d containers", len(nodes)*len(names))
recreateContainer := func(node *cluster.Node, confName string) {
defer wg.Done()
pool := node.Metadata["pool"]
containerConfig, confErr := nodecontainer.LoadNodeContainer(pool, confName)
if confErr != nil {
errChan <- confErr
return
}
if !containerConfig.Valid() {
return
}
log.Debugf("[node containers] recreating container %q in %s [%s]", confName, node.Address, pool)
fmt.Fprintf(w, "relaunching node container %q in the node %s [%s]\n", confName, node.Address, pool)
confErr = create(containerConfig, node, pool, p, relaunch)
if confErr != nil {
confErr = errors.Wrapf(confErr, "[node containers] failed to create container in %s [%s]", node.Address, pool)
errChan <- log.WrapError(confErr)
}
}
for i := range nodes {
wg.Add(1)
go func(node *cluster.Node) {
defer wg.Done()
for j := range names {
wg.Add(1)
go recreateContainer(node, names[j])
}
}(&nodes[i])
}
wg.Wait()
close(errChan)
var allErrors []error
for err = range errChan {
allErrors = append(allErrors, err)
}
if len(allErrors) == 0 {
return nil
}
return tsuruErrors.NewMultiError(allErrors...)
}
func (p *dockerProvisioner) MetricEnvs(app provision.App) map[string]string {
bsContainer, err := nodecontainer.LoadNodeContainer(app.GetPool(), nodecontainer.BsDefaultName)
if err != nil {
return map[string]string{}
}
envs := bsContainer.EnvMap()
for envName := range envs {
if !strings.HasPrefix(envName, "METRICS_") {
delete(envs, envName)
}
}
return envs
}
func (p *dockerProvisioner) LogsEnabled(app provision.App) (bool, string, error) {
const (
logBackendsEnv = "LOG_BACKENDS"
logDocKeyFormat = "LOG_%s_DOC"
tsuruLogBackendName = "tsuru"
)
isBS, err := container.LogIsBS(app.GetPool())
if err != nil {
return false, "", err
}
if !isBS {
driver, _, _ := container.LogOpts(app.GetPool())
msg := fmt.Sprintf("Logs not available through tsuru. Enabled log driver is %q.", driver)
return false, msg, nil
}
bsContainer, err := nodecontainer.LoadNodeContainer(app.GetPool(), nodecontainer.BsDefaultName)
if err != nil {
return false, "", err
}
envs := bsContainer.EnvMap()
enabledBackends := envs[logBackendsEnv]
if enabledBackends == "" {
return true, "", nil
}
backendsList := strings.Split(enabledBackends, ",")
for i := range backendsList {
backendsList[i] = strings.TrimSpace(backendsList[i])
if backendsList[i] == tsuruLogBackendName {
return true, "", nil
}
}
var docs []string
for _, backendName := range backendsList {
keyName := fmt.Sprintf(logDocKeyFormat, strings.ToUpper(backendName))
backendDoc := envs[keyName]
var docLine string
if backendDoc == "" {
docLine = fmt.Sprintf("* %s", backendName)
} else {
docLine = fmt.Sprintf("* %s: %s", backendName, backendDoc)
}
docs = append(docs, docLine)
}
fullDoc := fmt.Sprintf("Logs not available through tsuru. Enabled log backends are:\n%s",
strings.Join(docs, "\n"))
return false, fullDoc, nil
}
func migrateBSEnvs() error {
scheme, err := config.GetString("auth:scheme")
if err != nil {
scheme = nativeSchemeName
}
app.AuthScheme, err = auth.GetScheme(scheme)
if err != nil {
return err
}
_, err = nodecontainer.InitializeBS()
if err != nil {
return err
}
conn, err := db.Conn()
if err != nil {
return err
}
defer conn.Close()
var entry map[string]interface{}
err = conn.Collection("bsconfig").FindId("bs").One(&entry)
if err != nil {
if err == mgo.ErrNotFound {
return nil
}
return err
}
image, _ := entry["image"].(string)
envs, _ := entry["envs"].([]interface{})
var baseEnvs []string
for _, envEntry := range envs {
mapEntry, _ := envEntry.(map[string]interface{})
if mapEntry == nil {
continue
}
name, _ := mapEntry["name"].(string)
value, _ := mapEntry["value"].(string)
baseEnvs = append(baseEnvs, fmt.Sprintf("%s=%s", name, value))
}
bsNodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
if err != nil {
return err
}
if len(baseEnvs) > 0 {
bsNodeContainer.Config.Env = append(bsNodeContainer.Config.Env, baseEnvs...)
}
bsNodeContainer.PinnedImage = image
err = nodecontainer.AddNewContainer("", bsNodeContainer)
if err != nil {
return err
}
pools, _ := entry["pools"].([]interface{})
for _, poolData := range pools {
poolMap, _ := poolData.(map[string]interface{})
if poolMap == nil {
continue
}
poolName, _ := poolMap["name"].(string)
if poolName == "" {
continue
}
envs, _ := poolMap["envs"].([]interface{})
var toAdd []string
for _, envEntry := range envs {
mapEntry, _ := envEntry.(map[string]interface{})
if mapEntry == nil {
continue
}
name, _ := mapEntry["name"].(string)
value, _ := mapEntry["value"].(string)
toAdd = append(toAdd, fmt.Sprintf("%s=%s", name, value))
}
if len(toAdd) > 0 {
bsCont := nodecontainer.NodeContainerConfig{Name: nodecontainer.BsDefaultName}
bsCont.Config.Env = append(bsCont.Config.Env, toAdd...)
err = nodecontainer.AddNewContainer(poolName, &bsCont)
if err != nil {
return err
}
}
}
return nil
}
func (s *S) TestEnsureContainersStarted(c *check.C) {
c1 := nodecontainer.NodeContainerConfig{
Name: "bs",
Config: docker.Config{
Image: "bsimg",
Env: []string{
"A=1",
"B=2",
},
},
HostConfig: docker.HostConfig{
RestartPolicy: docker.AlwaysRestart(),
Privileged: true,
Binds: []string{"/xyz:/abc:rw"},
},
}
err := nodecontainer.AddNewContainer("", &c1)
c.Assert(err, check.IsNil)
c2 := c1
c2.Name = "sysdig"
c2.Config.Image = "sysdigimg"
c2.Config.Env = []string{"X=Z"}
err = nodecontainer.AddNewContainer("", &c2)
c.Assert(err, check.IsNil)
p, err := dockertest.StartMultipleServersCluster()
c.Assert(err, check.IsNil)
nodes, err := p.Cluster().Nodes()
c.Assert(err, check.IsNil)
for i, n := range nodes {
n.Metadata["pool"] = fmt.Sprintf("p-%d", i)
_, err = p.Cluster().UpdateNode(n)
c.Assert(err, check.IsNil)
}
var createBodies []string
var names []string
var mut sync.Mutex
server := p.Servers()[0]
server.CustomHandler("/containers/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mut.Lock()
defer mut.Unlock()
data, _ := ioutil.ReadAll(r.Body)
createBodies = append(createBodies, string(data))
names = append(names, r.URL.Query().Get("name"))
r.Body = ioutil.NopCloser(bytes.NewBuffer(data))
server.DefaultHandler().ServeHTTP(w, r)
}))
defer p.Destroy()
buf := safe.NewBuffer(nil)
err = ensureContainersStarted(p, buf, true, nil)
c.Assert(err, check.IsNil)
parts := strings.Split(buf.String(), "\n")
c.Assert(parts, check.HasLen, 5)
sort.Strings(parts)
c.Assert(parts[1], check.Matches, `relaunching node container "bs" in the node http://127.0.0.1:\d+/ \[p-0\]`)
c.Assert(parts[2], check.Matches, `relaunching node container "bs" in the node http://localhost:\d+/ \[p-1\]`)
c.Assert(parts[3], check.Matches, `relaunching node container "sysdig" in the node http://127.0.0.1:\d+/ \[p-0\]`)
c.Assert(parts[4], check.Matches, `relaunching node container "sysdig" in the node http://localhost:\d+/ \[p-1\]`)
c.Assert(createBodies, check.HasLen, 2)
c.Assert(names, check.HasLen, 2)
sort.Strings(names)
c.Assert(names, check.DeepEquals, []string{"bs", "sysdig"})
sort.Strings(createBodies)
result := make([]struct {
docker.Config
HostConfig docker.HostConfig
}, 2)
err = json.Unmarshal([]byte(createBodies[0]), &result[0])
c.Assert(err, check.IsNil)
err = json.Unmarshal([]byte(createBodies[1]), &result[1])
c.Assert(err, check.IsNil)
c.Assert(result, check.DeepEquals, []struct {
docker.Config
HostConfig docker.HostConfig
}{
{
Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "A=1", "B=2"}, Image: "bsimg",
Labels: map[string]string{
"tsuru.node.address": server.URL(),
"tsuru.node.pool": "p-0",
"tsuru.node.provisioner": "fake",
"tsuru.nodecontainer": "true",
}},
HostConfig: docker.HostConfig{
Binds: []string{"/xyz:/abc:rw"},
Privileged: true,
RestartPolicy: docker.RestartPolicy{Name: "always"},
LogConfig: docker.LogConfig{},
},
},
{
Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "X=Z"}, Image: "sysdigimg",
Labels: map[string]string{
"tsuru.node.address": server.URL(),
"tsuru.node.pool": "p-0",
"tsuru.node.provisioner": "fake",
"tsuru.nodecontainer": "true",
}},
HostConfig: docker.HostConfig{
Binds: []string{"/xyz:/abc:rw"},
Privileged: true,
RestartPolicy: docker.RestartPolicy{Name: "always"},
LogConfig: docker.LogConfig{},
},
},
})
nodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName)
c.Assert(err, check.IsNil)
c.Assert(nodeContainer.PinnedImage, check.Equals, "")
client, err := docker.NewClient(p.Servers()[0].URL())
c.Assert(err, check.IsNil)
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
client, err = docker.NewClient(p.Servers()[1].URL())
c.Assert(err, check.IsNil)
containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
}
func serviceSpecForNodeContainer(name, pool string) (*swarm.ServiceSpec, error) {
config, err := nodecontainer.LoadNodeContainer(pool, name)
if err != nil {
return nil, errors.WithStack(err)
}
var constraints []string
if pool == "" {
otherConfigs, err := nodecontainer.LoadNodeContainersForPools(name)
if err != nil {
return nil, errors.WithStack(err)
}
for p := range otherConfigs {
if p != "" {
constraints = append(constraints, fmt.Sprintf("node.labels.%s != %s", labelNodePoolName, p))
}
}
} else {
constraints = []string{fmt.Sprintf("node.labels.%s == %s", labelNodePoolName, pool)}
}
var mounts []mount.Mount
for _, b := range config.HostConfig.Binds {
parts := strings.SplitN(b, ":", 3)
mounts = append(mounts, mount.Mount{
Type: mount.TypeBind,
Source: parts[0],
Target: parts[1],
ReadOnly: parts[2] == "ro",
})
}
var healthcheck *container.HealthConfig
if config.Config.Healthcheck != nil {
healthcheck = &container.HealthConfig{
Test: config.Config.Healthcheck.Test,
Interval: config.Config.Healthcheck.Interval,
Timeout: config.Config.Healthcheck.Timeout,
Retries: config.Config.Healthcheck.Retries,
}
}
labels := config.Config.Labels
if labels == nil {
labels = make(map[string]string)
}
labels[labelNodeContainer.String()] = strconv.FormatBool(true)
labels[labelNodeContainerName.String()] = name
labels[labelPoolName.String()] = pool
labels[labelProvisionerName.String()] = "swarm"
service := &swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: nodeContainerServiceName(name, pool),
Labels: labels,
},
Mode: swarm.ServiceMode{Global: &swarm.GlobalService{}},
TaskTemplate: swarm.TaskSpec{
ContainerSpec: swarm.ContainerSpec{
Image: config.Image(),
Labels: labels,
Command: config.Config.Cmd,
Env: config.Config.Env,
Dir: config.Config.WorkingDir,
User: config.Config.User,
TTY: config.Config.Tty,
Mounts: mounts,
Healthcheck: healthcheck,
},
Placement: &swarm.Placement{Constraints: constraints},
},
}
return service, nil
}