func ensureContainersStarted(p DockerProvisioner, w io.Writer, relaunch bool, names []string, nodes ...cluster.Node) error {
if w == nil {
w = ioutil.Discard
}
var err error
if len(names) == 0 {
names, err = nodecontainer.AllNodeContainersNames()
if err != nil {
return err
}
}
if len(nodes) == 0 {
nodes, err = p.Cluster().UnfilteredNodes()
if err != nil {
return err
}
}
errChan := make(chan error, len(nodes)*len(names))
wg := sync.WaitGroup{}
log.Debugf("[node containers] recreating %d containers", len(nodes)*len(names))
recreateContainer := func(node *cluster.Node, confName string) {
defer wg.Done()
pool := node.Metadata["pool"]
containerConfig, confErr := nodecontainer.LoadNodeContainer(pool, confName)
if confErr != nil {
errChan <- confErr
return
}
if !containerConfig.Valid() {
return
}
log.Debugf("[node containers] recreating container %q in %s [%s]", confName, node.Address, pool)
fmt.Fprintf(w, "relaunching node container %q in the node %s [%s]\n", confName, node.Address, pool)
confErr = create(containerConfig, node, pool, p, relaunch)
if confErr != nil {
confErr = errors.Wrapf(confErr, "[node containers] failed to create container in %s [%s]", node.Address, pool)
errChan <- log.WrapError(confErr)
}
}
for i := range nodes {
wg.Add(1)
go func(node *cluster.Node) {
defer wg.Done()
for j := range names {
wg.Add(1)
go recreateContainer(node, names[j])
}
}(&nodes[i])
}
wg.Wait()
close(errChan)
var allErrors []error
for err = range errChan {
allErrors = append(allErrors, err)
}
if len(allErrors) == 0 {
return nil
}
return tsuruErrors.NewMultiError(allErrors...)
}
func (p *swarmProvisioner) Destroy(a provision.App) error {
client, err := chooseDBSwarmNode()
if err != nil {
return err
}
multiErrors := tsuruErrors.NewMultiError()
processes, err := allAppProcesses(a.GetName())
if err != nil {
multiErrors.Add(err)
}
for _, p := range processes {
name := serviceNameForApp(a, p)
err = client.RemoveService(docker.RemoveServiceOptions{
ID: name,
})
if err != nil {
if _, notFound := err.(*docker.NoSuchService); !notFound {
multiErrors.Add(errors.WithStack(err))
}
}
}
err = client.RemoveNetwork(networkNameForApp(a))
if err != nil {
multiErrors.Add(errors.WithStack(err))
}
if multiErrors.Len() > 0 {
return multiErrors
}
return nil
}
func RemoveNamedContainers(p DockerProvisioner, w io.Writer, name string, pool string) error {
var nodes []cluster.Node
var err error
if pool == "" {
nodes, err = p.Cluster().UnfilteredNodes()
} else {
nodes, err = p.Cluster().UnfilteredNodesForMetadata(map[string]string{"pool": pool})
}
if err != nil {
return errors.WithStack(err)
}
errChan := make(chan error, len(nodes))
wg := sync.WaitGroup{}
removeContainer := func(node *cluster.Node) {
pool := node.Metadata["pool"]
client, err := node.Client()
if err != nil {
errChan <- err
return
}
err = client.StopContainer(name, 10)
if err != nil {
if _, ok := err.(*docker.NoSuchContainer); ok {
log.Debugf("[node containers] no such container %q in %s [%s]", name, node.Address, pool)
fmt.Fprintf(w, "no such node container %q in the node %s [%s]\n", name, node.Address, pool)
return
}
if _, ok := err.(*docker.ContainerNotRunning); !ok {
err = errors.Wrapf(err, "[node containers] failed to stop container in %s [%s]", node.Address, pool)
errChan <- err
return
}
}
log.Debugf("[node containers] removing container %q in %s [%s]", name, node.Address, pool)
fmt.Fprintf(w, "removing node container %q in the node %s [%s]\n", name, node.Address, pool)
err = client.RemoveContainer(docker.RemoveContainerOptions{ID: name, Force: true})
if err != nil {
err = errors.Wrapf(err, "[node containers] failed to remove container in %s [%s]", node.Address, pool)
errChan <- err
}
}
for i := range nodes {
wg.Add(1)
go func(node *cluster.Node) {
removeContainer(node)
wg.Done()
}(&nodes[i])
}
wg.Wait()
close(errChan)
var allErrors []error
for err := range errChan {
allErrors = append(allErrors, err)
}
if len(allErrors) == 0 {
return nil
}
return tsuruErrors.NewMultiError(allErrors...)
}
func (g MultiGuesser) GuessName(pathname string) (string, error) {
var errors []error
for _, guesser := range g.Guessers {
app, err := guesser.GuessName(pathname)
if err == nil {
return app, nil
}
errors = append(errors, err)
}
return "", tsuruErrors.NewMultiError(errors...)
}
func create(c *nodecontainer.NodeContainerConfig, node *cluster.Node, poolName string, p DockerProvisioner, relaunch bool) error {
client, err := node.Client()
if err != nil {
return err
}
c.Config.Image, err = pullImage(c, client, p, poolName)
if err != nil {
return err
}
c.Config.Env = append([]string{"DOCKER_ENDPOINT=" + node.Address}, c.Config.Env...)
if c.Config.Labels == nil {
c.Config.Labels = map[string]string{}
}
c.Config.Labels["tsuru.nodecontainer"] = strconv.FormatBool(true)
c.Config.Labels["tsuru.node.pool"] = poolName
c.Config.Labels["tsuru.node.address"] = node.Address
c.Config.Labels["tsuru.node.provisioner"] = p.GetName()
opts := docker.CreateContainerOptions{
Name: c.Name,
HostConfig: &c.HostConfig,
Config: &c.Config,
}
_, err = client.CreateContainer(opts)
if err != nil {
if err != docker.ErrContainerAlreadyExists {
return err
}
if relaunch {
multiErr := tsuruErrors.NewMultiError()
err = tryRemovingOld(client, opts.Name)
if err != nil {
multiErr.Add(errors.Wrapf(err, "unable to remove old node-container"))
}
_, err = client.CreateContainer(opts)
if err != nil {
multiErr.Add(errors.Wrapf(err, "unable to create new node-container"))
return multiErr
}
}
}
err = client.StartContainer(c.Name, nil)
if _, ok := err.(*docker.ContainerAlreadyRunning); !ok {
return err
}
return nil
}
// PlatformAdd add a new platform to tsuru
func PlatformAdd(opts provision.PlatformOptions) error {
if opts.Name == "" {
return ErrPlatformNameMissing
}
provisioners, err := provision.Registry()
if err != nil {
return err
}
p := Platform{Name: opts.Name}
conn, err := db.Conn()
if err != nil {
return err
}
defer conn.Close()
err = conn.Platforms().Insert(p)
if err != nil {
if mgo.IsDup(err) {
return DuplicatePlatformError
}
return err
}
for _, p := range provisioners {
if extensibleProv, ok := p.(provision.ExtensibleProvisioner); ok {
err = extensibleProv.PlatformAdd(opts)
if err != nil {
break
}
}
}
if err != nil {
dbErr := conn.Platforms().RemoveId(p.Name)
if dbErr != nil {
return tsuruErrors.NewMultiError(
errors.Wrapf(dbErr, "unable to rollback platform add"),
errors.Wrapf(err, "original platform add error"),
)
}
return err
}
return nil
}
func (p *swarmProvisioner) UpgradeNodeContainer(name string, pool string, writer io.Writer) error {
client, err := chooseDBSwarmNode()
if err != nil {
if errors.Cause(err) == errNoSwarmNode {
return nil
}
return err
}
poolsToRun := []string{pool}
if pool == "" {
poolMap, errLoad := nodecontainer.LoadNodeContainersForPools(name)
if errLoad != nil {
return errors.WithStack(errLoad)
}
poolsToRun = make([]string, len(poolMap))
i := 0
for k, v := range poolMap {
if !v.Valid() {
continue
}
if k == "" {
poolsToRun[len(poolMap)-1] = k
continue
}
poolsToRun[i] = k
i++
}
}
var allErrors []error
created := false
for _, v := range poolsToRun {
serviceSpec, errUpsert := serviceSpecForNodeContainer(name, v)
if errUpsert != nil {
errUpsert = errors.Wrapf(errUpsert, "[node containers] failed retrieve service spec for node container %q [%s]", name, v)
allErrors = append(allErrors, errUpsert)
}
log.Debugf("[node containers] upserting service %q for node container %s [%s]", serviceSpec.Name, name, v)
fmt.Fprintf(writer, "upserting service %q for node container %q [%s]\n", serviceSpec.Name, name, v)
errUpsert, created = upsertService(serviceSpec, client)
if errUpsert != nil {
errUpsert = errors.Wrapf(errUpsert, "[node containers] failed upsert service %q for node container %q [%s]", serviceSpec.Name, name, v)
allErrors = append(allErrors, errUpsert)
}
}
if pool != "" && created {
serviceName := nodeContainerServiceName(name, "")
baseSpec, err := client.InspectService(serviceName)
if err != nil {
if _, ok := err.(*docker.NoSuchService); ok {
return nil
}
err = errors.Wrapf(err, "[node containers] failed inspect base service %q for node container %q", serviceName, name)
allErrors = append(allErrors, err)
}
newBaseSpec, err := serviceSpecForNodeContainer(name, "")
if err != nil {
err = errors.Wrapf(err, "[node containers] failed retrieve base service spec %q for node container %q", serviceName, name)
allErrors = append(allErrors, err)
}
baseSpec.Spec.TaskTemplate.Placement = newBaseSpec.TaskTemplate.Placement
log.Debugf("[node containers] updating base service %q for node container %s constraints", serviceName, name)
fmt.Fprintf(writer, "updating base service %q for node container %s constraints\n", serviceName, name)
err, _ = upsertService(&baseSpec.Spec, client)
if err != nil {
err = errors.Wrapf(err, "[node containers] failed update base service %q for node container %q", serviceName, name)
allErrors = append(allErrors, err)
}
}
if len(allErrors) == 0 {
return nil
}
return tsuruErrors.NewMultiError(allErrors...)
}
func (i *dockerMachineIaaS) CreateMachine(params map[string]string) (*iaas.Machine, error) {
caPath, _ := i.base.GetConfigString("ca-path")
driverName, ok := params["driver"]
if !ok {
name, errConf := i.base.GetConfigString("driver:name")
if errConf != nil {
return nil, errDriverNotSet
}
driverName = name
params["driver"] = driverName
}
dockerEngineInstallURL, _ := i.getParamOrConfigString("docker-install-url", params)
insecureRegistry, _ := i.getParamOrConfigString("insecure-registry", params)
var engineFlags []string
if f, err := i.getParamOrConfigString("docker-flags", params); err == nil {
engineFlags = strings.Split(f, ",")
}
machineName, ok := params["name"]
if !ok {
name, err := generateMachineName(params["pool"])
if err != nil {
return nil, err
}
machineName = name
} else {
delete(params, "name")
}
userDataFileParam, err := i.base.GetConfigString("driver:user-data-file-param")
if err == nil {
f, errTemp := ioutil.TempFile("", "")
if errTemp != nil {
return nil, errors.Wrap(errTemp, "failed to create userdata file")
}
defer os.RemoveAll(f.Name())
userData, errData := i.base.ReadUserData()
if errData != nil {
return nil, errors.WithMessage(errData, "failed to read userdata")
}
_, errWrite := f.WriteString(userData)
if errWrite != nil {
return nil, errors.Wrap(errWrite, "failed to write local userdata file")
}
params[userDataFileParam] = f.Name()
}
driverOpts := i.buildDriverOpts(driverName, params)
if userDataFileParam != "" {
delete(params, userDataFileParam)
}
buf := &bytes.Buffer{}
debugConf, _ := i.base.GetConfigString("debug")
if debugConf == "" {
debugConf = "false"
}
isDebug, err := strconv.ParseBool(debugConf)
if err != nil {
return nil, errors.Wrap(err, "failed to parse debug config")
}
dockerMachine, err := i.apiFactory(DockerMachineConfig{
CaPath: caPath,
OutWriter: buf,
ErrWriter: buf,
IsDebug: isDebug,
})
if err != nil {
return nil, err
}
defer func() {
dockerMachine.Close()
log.Debug(buf.String())
}()
m, err := dockerMachine.CreateMachine(CreateMachineOpts{
Name: machineName,
DriverName: driverName,
Params: driverOpts,
InsecureRegistry: insecureRegistry,
DockerEngineInstallURL: dockerEngineInstallURL,
ArbitraryFlags: engineFlags,
})
if err != nil {
if m != nil {
errRem := dockerMachine.DeleteMachine(m.Base)
if errRem != nil {
err = tsuruErrors.NewMultiError(err, errors.WithMessage(errRem, "failed to remove machine after error"))
}
}
return nil, err
}
m.Base.CreationParams = params
return m.Base, nil
}