func (s *S) TestEnsureContainersStartedAlreadyPinned(c *check.C) { config.Set("docker:bs:image", "myregistry/tsuru/bs") _, err := nodecontainer.InitializeBS() c.Assert(err, check.IsNil) cont, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) c.Assert(err, check.IsNil) cont.PinnedImage = "myregistry/tsuru/bs@" + digest err = nodecontainer.AddNewContainer("", cont) c.Assert(err, check.IsNil) server, err := testing.NewServer("127.0.0.1:0", nil, nil) c.Assert(err, check.IsNil) defer server.Stop() server.CustomHandler("/images/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) server.DefaultHandler().ServeHTTP(w, r) w.Write([]byte(pullOutputDigest)) })) p, err := dockertest.NewFakeDockerProvisioner(server.URL()) c.Assert(err, check.IsNil) defer p.Destroy() client, err := docker.NewClient(server.URL()) c.Assert(err, check.IsNil) err = client.PullImage(docker.PullImageOptions{ Repository: "base", }, docker.AuthConfiguration{}) c.Assert(err, check.IsNil) _, err = client.CreateContainer(docker.CreateContainerOptions{ Name: nodecontainer.BsDefaultName, Config: &docker.Config{Image: "base"}, HostConfig: &docker.HostConfig{}, }) c.Assert(err, check.IsNil) buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 1) container, err := client.InspectContainer(containers[0].ID) c.Assert(err, check.IsNil) c.Assert(container.Name, check.Equals, nodecontainer.BsDefaultName) c.Assert(container.Config.Image, check.Equals, "myregistry/tsuru/bs@"+digest) c.Assert(container.HostConfig.RestartPolicy, check.Equals, docker.AlwaysRestart()) c.Assert(container.State.Running, check.Equals, true) nodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) c.Assert(err, check.IsNil) c.Assert(nodeContainer.PinnedImage, check.Equals, "myregistry/tsuru/bs@"+digest) }
func ensureContainersStarted(p DockerProvisioner, w io.Writer, relaunch bool, names []string, nodes ...cluster.Node) error { if w == nil { w = ioutil.Discard } var err error if len(names) == 0 { names, err = nodecontainer.AllNodeContainersNames() if err != nil { return err } } if len(nodes) == 0 { nodes, err = p.Cluster().UnfilteredNodes() if err != nil { return err } } errChan := make(chan error, len(nodes)*len(names)) wg := sync.WaitGroup{} log.Debugf("[node containers] recreating %d containers", len(nodes)*len(names)) recreateContainer := func(node *cluster.Node, confName string) { defer wg.Done() pool := node.Metadata["pool"] containerConfig, confErr := nodecontainer.LoadNodeContainer(pool, confName) if confErr != nil { errChan <- confErr return } if !containerConfig.Valid() { return } log.Debugf("[node containers] recreating container %q in %s [%s]", confName, node.Address, pool) fmt.Fprintf(w, "relaunching node container %q in the node %s [%s]\n", confName, node.Address, pool) confErr = create(containerConfig, node, pool, p, relaunch) if confErr != nil { confErr = errors.Wrapf(confErr, "[node containers] failed to create container in %s [%s]", node.Address, pool) errChan <- log.WrapError(confErr) } } for i := range nodes { wg.Add(1) go func(node *cluster.Node) { defer wg.Done() for j := range names { wg.Add(1) go recreateContainer(node, names[j]) } }(&nodes[i]) } wg.Wait() close(errChan) var allErrors []error for err = range errChan { allErrors = append(allErrors, err) } if len(allErrors) == 0 { return nil } return tsuruErrors.NewMultiError(allErrors...) }
func (p *dockerProvisioner) MetricEnvs(app provision.App) map[string]string { bsContainer, err := nodecontainer.LoadNodeContainer(app.GetPool(), nodecontainer.BsDefaultName) if err != nil { return map[string]string{} } envs := bsContainer.EnvMap() for envName := range envs { if !strings.HasPrefix(envName, "METRICS_") { delete(envs, envName) } } return envs }
func (p *dockerProvisioner) LogsEnabled(app provision.App) (bool, string, error) { const ( logBackendsEnv = "LOG_BACKENDS" logDocKeyFormat = "LOG_%s_DOC" tsuruLogBackendName = "tsuru" ) isBS, err := container.LogIsBS(app.GetPool()) if err != nil { return false, "", err } if !isBS { driver, _, _ := container.LogOpts(app.GetPool()) msg := fmt.Sprintf("Logs not available through tsuru. Enabled log driver is %q.", driver) return false, msg, nil } bsContainer, err := nodecontainer.LoadNodeContainer(app.GetPool(), nodecontainer.BsDefaultName) if err != nil { return false, "", err } envs := bsContainer.EnvMap() enabledBackends := envs[logBackendsEnv] if enabledBackends == "" { return true, "", nil } backendsList := strings.Split(enabledBackends, ",") for i := range backendsList { backendsList[i] = strings.TrimSpace(backendsList[i]) if backendsList[i] == tsuruLogBackendName { return true, "", nil } } var docs []string for _, backendName := range backendsList { keyName := fmt.Sprintf(logDocKeyFormat, strings.ToUpper(backendName)) backendDoc := envs[keyName] var docLine string if backendDoc == "" { docLine = fmt.Sprintf("* %s", backendName) } else { docLine = fmt.Sprintf("* %s: %s", backendName, backendDoc) } docs = append(docs, docLine) } fullDoc := fmt.Sprintf("Logs not available through tsuru. Enabled log backends are:\n%s", strings.Join(docs, "\n")) return false, fullDoc, nil }
func migrateBSEnvs() error { scheme, err := config.GetString("auth:scheme") if err != nil { scheme = nativeSchemeName } app.AuthScheme, err = auth.GetScheme(scheme) if err != nil { return err } _, err = nodecontainer.InitializeBS() if err != nil { return err } conn, err := db.Conn() if err != nil { return err } defer conn.Close() var entry map[string]interface{} err = conn.Collection("bsconfig").FindId("bs").One(&entry) if err != nil { if err == mgo.ErrNotFound { return nil } return err } image, _ := entry["image"].(string) envs, _ := entry["envs"].([]interface{}) var baseEnvs []string for _, envEntry := range envs { mapEntry, _ := envEntry.(map[string]interface{}) if mapEntry == nil { continue } name, _ := mapEntry["name"].(string) value, _ := mapEntry["value"].(string) baseEnvs = append(baseEnvs, fmt.Sprintf("%s=%s", name, value)) } bsNodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) if err != nil { return err } if len(baseEnvs) > 0 { bsNodeContainer.Config.Env = append(bsNodeContainer.Config.Env, baseEnvs...) } bsNodeContainer.PinnedImage = image err = nodecontainer.AddNewContainer("", bsNodeContainer) if err != nil { return err } pools, _ := entry["pools"].([]interface{}) for _, poolData := range pools { poolMap, _ := poolData.(map[string]interface{}) if poolMap == nil { continue } poolName, _ := poolMap["name"].(string) if poolName == "" { continue } envs, _ := poolMap["envs"].([]interface{}) var toAdd []string for _, envEntry := range envs { mapEntry, _ := envEntry.(map[string]interface{}) if mapEntry == nil { continue } name, _ := mapEntry["name"].(string) value, _ := mapEntry["value"].(string) toAdd = append(toAdd, fmt.Sprintf("%s=%s", name, value)) } if len(toAdd) > 0 { bsCont := nodecontainer.NodeContainerConfig{Name: nodecontainer.BsDefaultName} bsCont.Config.Env = append(bsCont.Config.Env, toAdd...) err = nodecontainer.AddNewContainer(poolName, &bsCont) if err != nil { return err } } } return nil }
func (s *S) TestEnsureContainersStarted(c *check.C) { c1 := nodecontainer.NodeContainerConfig{ Name: "bs", Config: docker.Config{ Image: "bsimg", Env: []string{ "A=1", "B=2", }, }, HostConfig: docker.HostConfig{ RestartPolicy: docker.AlwaysRestart(), Privileged: true, Binds: []string{"/xyz:/abc:rw"}, }, } err := nodecontainer.AddNewContainer("", &c1) c.Assert(err, check.IsNil) c2 := c1 c2.Name = "sysdig" c2.Config.Image = "sysdigimg" c2.Config.Env = []string{"X=Z"} err = nodecontainer.AddNewContainer("", &c2) c.Assert(err, check.IsNil) p, err := dockertest.StartMultipleServersCluster() c.Assert(err, check.IsNil) nodes, err := p.Cluster().Nodes() c.Assert(err, check.IsNil) for i, n := range nodes { n.Metadata["pool"] = fmt.Sprintf("p-%d", i) _, err = p.Cluster().UpdateNode(n) c.Assert(err, check.IsNil) } var createBodies []string var names []string var mut sync.Mutex server := p.Servers()[0] server.CustomHandler("/containers/create", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mut.Lock() defer mut.Unlock() data, _ := ioutil.ReadAll(r.Body) createBodies = append(createBodies, string(data)) names = append(names, r.URL.Query().Get("name")) r.Body = ioutil.NopCloser(bytes.NewBuffer(data)) server.DefaultHandler().ServeHTTP(w, r) })) defer p.Destroy() buf := safe.NewBuffer(nil) err = ensureContainersStarted(p, buf, true, nil) c.Assert(err, check.IsNil) parts := strings.Split(buf.String(), "\n") c.Assert(parts, check.HasLen, 5) sort.Strings(parts) c.Assert(parts[1], check.Matches, `relaunching node container "bs" in the node http://127.0.0.1:\d+/ \[p-0\]`) c.Assert(parts[2], check.Matches, `relaunching node container "bs" in the node http://localhost:\d+/ \[p-1\]`) c.Assert(parts[3], check.Matches, `relaunching node container "sysdig" in the node http://127.0.0.1:\d+/ \[p-0\]`) c.Assert(parts[4], check.Matches, `relaunching node container "sysdig" in the node http://localhost:\d+/ \[p-1\]`) c.Assert(createBodies, check.HasLen, 2) c.Assert(names, check.HasLen, 2) sort.Strings(names) c.Assert(names, check.DeepEquals, []string{"bs", "sysdig"}) sort.Strings(createBodies) result := make([]struct { docker.Config HostConfig docker.HostConfig }, 2) err = json.Unmarshal([]byte(createBodies[0]), &result[0]) c.Assert(err, check.IsNil) err = json.Unmarshal([]byte(createBodies[1]), &result[1]) c.Assert(err, check.IsNil) c.Assert(result, check.DeepEquals, []struct { docker.Config HostConfig docker.HostConfig }{ { Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "A=1", "B=2"}, Image: "bsimg", Labels: map[string]string{ "tsuru.node.address": server.URL(), "tsuru.node.pool": "p-0", "tsuru.node.provisioner": "fake", "tsuru.nodecontainer": "true", }}, HostConfig: docker.HostConfig{ Binds: []string{"/xyz:/abc:rw"}, Privileged: true, RestartPolicy: docker.RestartPolicy{Name: "always"}, LogConfig: docker.LogConfig{}, }, }, { Config: docker.Config{Env: []string{"DOCKER_ENDPOINT=" + server.URL(), "X=Z"}, Image: "sysdigimg", Labels: map[string]string{ "tsuru.node.address": server.URL(), "tsuru.node.pool": "p-0", "tsuru.node.provisioner": "fake", "tsuru.nodecontainer": "true", }}, HostConfig: docker.HostConfig{ Binds: []string{"/xyz:/abc:rw"}, Privileged: true, RestartPolicy: docker.RestartPolicy{Name: "always"}, LogConfig: docker.LogConfig{}, }, }, }) nodeContainer, err := nodecontainer.LoadNodeContainer("", nodecontainer.BsDefaultName) c.Assert(err, check.IsNil) c.Assert(nodeContainer.PinnedImage, check.Equals, "") client, err := docker.NewClient(p.Servers()[0].URL()) c.Assert(err, check.IsNil) containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) client, err = docker.NewClient(p.Servers()[1].URL()) c.Assert(err, check.IsNil) containers, err = client.ListContainers(docker.ListContainersOptions{All: true}) c.Assert(err, check.IsNil) c.Assert(containers, check.HasLen, 2) }
func serviceSpecForNodeContainer(name, pool string) (*swarm.ServiceSpec, error) { config, err := nodecontainer.LoadNodeContainer(pool, name) if err != nil { return nil, errors.WithStack(err) } var constraints []string if pool == "" { otherConfigs, err := nodecontainer.LoadNodeContainersForPools(name) if err != nil { return nil, errors.WithStack(err) } for p := range otherConfigs { if p != "" { constraints = append(constraints, fmt.Sprintf("node.labels.%s != %s", labelNodePoolName, p)) } } } else { constraints = []string{fmt.Sprintf("node.labels.%s == %s", labelNodePoolName, pool)} } var mounts []mount.Mount for _, b := range config.HostConfig.Binds { parts := strings.SplitN(b, ":", 3) mounts = append(mounts, mount.Mount{ Type: mount.TypeBind, Source: parts[0], Target: parts[1], ReadOnly: parts[2] == "ro", }) } var healthcheck *container.HealthConfig if config.Config.Healthcheck != nil { healthcheck = &container.HealthConfig{ Test: config.Config.Healthcheck.Test, Interval: config.Config.Healthcheck.Interval, Timeout: config.Config.Healthcheck.Timeout, Retries: config.Config.Healthcheck.Retries, } } labels := config.Config.Labels if labels == nil { labels = make(map[string]string) } labels[labelNodeContainer.String()] = strconv.FormatBool(true) labels[labelNodeContainerName.String()] = name labels[labelPoolName.String()] = pool labels[labelProvisionerName.String()] = "swarm" service := &swarm.ServiceSpec{ Annotations: swarm.Annotations{ Name: nodeContainerServiceName(name, pool), Labels: labels, }, Mode: swarm.ServiceMode{Global: &swarm.GlobalService{}}, TaskTemplate: swarm.TaskSpec{ ContainerSpec: swarm.ContainerSpec{ Image: config.Image(), Labels: labels, Command: config.Config.Cmd, Env: config.Config.Env, Dir: config.Config.WorkingDir, User: config.Config.User, TTY: config.Config.Tty, Mounts: mounts, Healthcheck: healthcheck, }, Placement: &swarm.Placement{Constraints: constraints}, }, } return service, nil }