Esempio n. 1
0
// LoadImage checks the client for an image matching from. If not found,
// attempts to pull the image and then tries to inspect again.
func (e *ClientExecutor) LoadImage(from string) (*docker.Image, error) {
	image, err := e.Client.InspectImage(from)
	if err == nil {
		return image, nil
	}
	if err != docker.ErrNoSuchImage {
		return nil, err
	}

	if !e.AllowPull {
		glog.V(4).Infof("image %s did not exist", from)
		return nil, docker.ErrNoSuchImage
	}

	repository, tag := docker.ParseRepositoryTag(from)
	if len(tag) == 0 {
		tag = "latest"
	}

	glog.V(4).Infof("attempting to pull %s with auth from repository %s:%s", from, repository, tag)

	// TODO: we may want to abstract looping over multiple credentials
	auth, _ := e.AuthFn(repository)
	if len(auth) == 0 {
		auth = append(auth, credentialprovider.LazyAuthConfiguration{})
	}

	if e.LogFn != nil {
		e.LogFn("Image %s was not found, pulling ...", from)
	}

	var lastErr error
	outputProgress := func(s string) {
		e.LogFn("%s", s)
	}
	for _, config := range auth {
		// TODO: handle IDs?
		pullImageOptions := docker.PullImageOptions{
			Repository:    repository,
			Tag:           tag,
			OutputStream:  imageprogress.NewPullWriter(outputProgress),
			RawJSONStream: true,
		}
		if glog.V(5) {
			pullImageOptions.OutputStream = os.Stderr
			pullImageOptions.RawJSONStream = false
		}
		authConfig := docker.AuthConfiguration{Username: config.Username, ServerAddress: config.ServerAddress, Password: config.Password}
		if err = e.Client.PullImage(pullImageOptions, authConfig); err == nil {
			break
		}
		lastErr = err
		continue
	}
	if lastErr != nil {
		return nil, fmt.Errorf("unable to pull image (from: %s, tag: %s): %v", repository, tag, lastErr)
	}

	return e.Client.InspectImage(from)
}
Esempio n. 2
0
// CommitContainer commits a container to an image with a specific tag.
// The new image ID is returned
func (d *stiDocker) CommitContainer(opts CommitContainerOptions) (string, error) {
	repository, tag := docker.ParseRepositoryTag(opts.Repository)
	dockerOpts := docker.CommitContainerOptions{
		Container:  opts.ContainerID,
		Repository: repository,
		Tag:        tag,
	}
	if opts.Command != nil || opts.Entrypoint != nil {
		config := docker.Config{
			Cmd:        opts.Command,
			Entrypoint: opts.Entrypoint,
			Env:        opts.Env,
			Labels:     opts.Labels,
			User:       opts.User,
		}
		dockerOpts.Run = &config
		glog.V(2).Infof("Committing container with dockerOpts: %+v, config: %+v", dockerOpts, config)
	}

	image, err := d.client.CommitContainer(dockerOpts)
	if err == nil && image != nil {
		return image.ID, nil
	}
	return "", err
}
Esempio n. 3
0
// pushImage pushes a docker image to the registry specified in its tag.
// The method will retry to push the image when following scenarios occur:
// - Docker registry is down temporarily or permanently
// - other image is being pushed to the registry
// If any other scenario the push will fail, without retries.
func pushImage(client DockerClient, name string, authConfig docker.AuthConfiguration) error {
	repository, tag := docker.ParseRepositoryTag(name)
	opts := docker.PushImageOptions{
		Name: repository,
		Tag:  tag,
	}
	if glog.V(5) {
		opts.OutputStream = os.Stderr
	}
	var err error

	for retries := 0; retries <= DefaultPushRetryCount; retries++ {
		err = client.PushImage(opts, authConfig)
		if err == nil {
			return nil
		}

		errMsg := fmt.Sprintf("%s", err)
		if !strings.Contains(errMsg, "ping attempt failed with error") && !strings.Contains(errMsg, "is already in progress") {
			return err
		}

		util.HandleError(fmt.Errorf("push for image %s failed, will retry in %s seconds ...", name, DefaultPushRetryDelay))
		glog.Flush()
		time.Sleep(DefaultPushRetryDelay)
	}
	return err
}
Esempio n. 4
0
// pushImage pushes a docker image to the registry specified in its tag.
// The method will retry to push the image when following scenarios occur:
// - Docker registry is down temporarily or permanently
// - other image is being pushed to the registry
// If any other scenario the push will fail, without retries.
func pushImage(client DockerClient, name string, authConfig docker.AuthConfiguration) error {
	repository, tag := docker.ParseRepositoryTag(name)
	opts := docker.PushImageOptions{
		Name: repository,
		Tag:  tag,
	}
	if glog.V(5) {
		opts.OutputStream = os.Stderr
	}
	var err error
	var retriableError = false

	for retries := 0; retries <= DefaultPushRetryCount; retries++ {
		err = client.PushImage(opts, authConfig)
		if err == nil {
			return nil
		}

		errMsg := fmt.Sprintf("%s", err)
		for _, errorString := range RetriableErrors {
			if strings.Contains(errMsg, errorString) {
				retriableError = true
				break
			}
		}
		if !retriableError {
			return err
		}

		utilruntime.HandleError(fmt.Errorf("push for image %s failed, will retry in %s ...", name, DefaultPushRetryDelay))
		glog.Flush()
		time.Sleep(DefaultPushRetryDelay)
	}
	return err
}
Esempio n. 5
0
// createImage creates a docker image either by pulling it from a registry or by
// loading it from the file system
func (d *DockerDriver) createImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir string) error {
	image := driverConfig.ImageName
	repo, tag := docker.ParseRepositoryTag(image)
	if tag == "" {
		tag = "latest"
	}

	var dockerImage *docker.Image
	var err error
	// We're going to check whether the image is already downloaded. If the tag
	// is "latest" we have to check for a new version every time so we don't
	// bother to check and cache the id here. We'll download first, then cache.
	if tag != "latest" {
		dockerImage, err = client.InspectImage(image)
	}

	// Download the image
	if dockerImage == nil {
		if len(driverConfig.LoadImages) > 0 {
			return d.loadImage(driverConfig, client, taskDir)
		}

		return d.pullImage(driverConfig, client, repo, tag)
	}
	return err
}
Esempio n. 6
0
// pullImage pulls the latest image for the given repotag from a public
// registry.
func (c *client) pullImage(repoTag string) error {
	// configuration options that get passed to client.PullImage
	repository, tag := docker.ParseRepositoryTag(repoTag)
	pullImageOptions := docker.PullImageOptions{Repository: repository, Tag: tag}
	// pull image from registry
	return c.client.PullImage(pullImageOptions, docker.AuthConfiguration{})
}
Esempio n. 7
0
// getImageName checks the image name and adds DefaultTag if none is specified
func getImageName(name string) string {
	_, tag := docker.ParseRepositoryTag(name)
	if len(tag) == 0 {
		return strings.Join([]string{name, DefaultTag}, ":")
	}

	return name
}
Esempio n. 8
0
// ValidateImage validates the image field does not include a tag
func (c *ImageConfig) ValidateImage() error {
	_, tag := docker.ParseRepositoryTag(c.Image)
	if tag != "" {
		return fmt.Errorf(
			"tag %q must be specified in the `tags` field, not in `image`", tag)
	}
	return nil
}
Esempio n. 9
0
// ValidateTags to ensure the first tag is a basic tag without an image name.
func (c *ImageConfig) ValidateTags() error {
	if len(c.Tags) == 0 {
		return nil
	}
	_, tag := docker.ParseRepositoryTag(c.Tags[0])
	if tag != "" {
		return fmt.Errorf("the first tag %q may not include an image name", tag)
	}
	return nil

}
Esempio n. 10
0
// tagImage uses the dockerClient to tag a Docker image with name. It is a
// helper to facilitate the usage of dockerClient.TagImage, because the former
// requires the name to be split into more explicit parts.
func tagImage(dockerClient DockerClient, image, name string) error {
	repo, tag := docker.ParseRepositoryTag(name)
	return dockerClient.TagImage(image, docker.TagImageOptions{
		Repo: repo,
		Tag:  tag,
		// We need to set Force to true to update the tag even if it
		// already exists. This is the same behavior as `docker build -t
		// tag .`.
		Force: true,
	})
}
Esempio n. 11
0
File: tag.go Progetto: dnephin/dobi
func tagImage(ctx *context.ExecuteContext, config *config.ImageConfig, imageTag string) error {
	canonicalImageTag := GetImageName(ctx, config)
	if imageTag == canonicalImageTag {
		return nil
	}

	repo, tag := docker.ParseRepositoryTag(imageTag)
	err := ctx.Client.TagImage(canonicalImageTag, docker.TagImageOptions{
		Repo:  repo,
		Tag:   tag,
		Force: true,
	})
	if err != nil {
		return fmt.Errorf("failed to add tag %q: %s", imageTag, err)
	}
	return nil
}
Esempio n. 12
0
File: pull.go Progetto: dnephin/dobi
func pullImage(ctx *context.ExecuteContext, t *Task, imageTag string) error {
	registry, err := parseAuthRepo(t.config.Image)
	if err != nil {
		return err
	}

	repo, tag := docker.ParseRepositoryTag(imageTag)
	return Stream(os.Stdout, func(out io.Writer) error {
		return ctx.Client.PullImage(docker.PullImageOptions{
			Repository:    repo,
			Tag:           tag,
			OutputStream:  out,
			RawJSONStream: true,
			// TODO: timeout
		}, ctx.GetAuthConfig(registry))
	})
}
Esempio n. 13
0
func importImage(client *dockerClient.Client, name, fileName string) error {
	file, err := os.Open(fileName)
	if err != nil {
		return err
	}

	defer file.Close()

	log.Debugf("Importing image for %s", fileName)
	repo, tag := dockerClient.ParseRepositoryTag(name)
	return client.ImportImage(dockerClient.ImportImageOptions{
		Source:      "-",
		Repository:  repo,
		Tag:         tag,
		InputStream: file,
	})
}
Esempio n. 14
0
// ForEachTag runs a function for each tag
func (t *Task) ForEachTag(ctx *context.ExecuteContext, each func(string) error) error {
	if len(t.config.Tags) == 0 {
		return each(GetImageName(ctx, t.config))
	}

	for _, tag := range t.config.Tags {
		imageTag := t.config.Image + ":" + tag
		// If the tag is already a complete image name then use it directly
		if _, hasTag := docker.ParseRepositoryTag(tag); hasTag != "" {
			imageTag = tag
		}

		if err := each(imageTag); err != nil {
			return err
		}
	}
	return nil
}
Esempio n. 15
0
// LoadImage checks the client for an image matching from. If not found,
// attempts to pull the image and then tries to inspect again.
func (e *ClientExecutor) LoadImage(from string) (*docker.Image, error) {
	image, err := e.Client.InspectImage(from)
	if err == nil {
		return image, nil
	}
	if err != docker.ErrNoSuchImage {
		return nil, err
	}

	if !e.AllowPull {
		glog.V(4).Infof("image %s did not exist", from)
		return nil, docker.ErrNoSuchImage
	}

	repository, _ := docker.ParseRepositoryTag(from)

	glog.V(4).Infof("attempting to pull %s with auth from repository %s", from, repository)

	// TODO: we may want to abstract looping over multiple credentials
	auth, _ := e.AuthFn(repository)
	if len(auth) == 0 {
		auth = append(auth, docker.AuthConfiguration{})
	}

	if e.LogFn != nil {
		e.LogFn("Image %s was not found, pulling ...", from)
	}

	var lastErr error
	for _, config := range auth {
		// TODO: handle IDs?
		// TODO: use RawJSONStream:true and handle the output nicely
		if err = e.Client.PullImage(docker.PullImageOptions{Repository: from, OutputStream: e.Out}, config); err == nil {
			break
		}
		lastErr = err
		continue
	}
	if lastErr != nil {
		return nil, lastErr
	}

	return e.Client.InspectImage(from)
}
Esempio n. 16
0
// GetAuthconfig retrieves the correct auth configuration for the given repository
func (authProvider *dockerAuthProvider) GetAuthconfig(image string) (docker.AuthConfiguration, error) {
	// Ignore 'tag', not used in auth determination
	repository, _ := docker.ParseRepositoryTag(image)
	authDataMap := authProvider.authMap

	// Ignore repo/image name for some auth checks (see use of 'image' below for where it's not ignored.
	indexName, _ := splitReposName(repository)

	if isDockerhubHostname(indexName) {
		return authDataMap[dockerRegistryKey], nil
	}

	// Try to find the longest match that at least matches the hostname
	// This is to support e.g. 'registry.tld: auth1, registry.tld/username:
	// auth2' for multiple different auths on the same registry.
	longestKey := ""
	authConfigKey := indexName

	// Take a direct match of the index hostname as a sane default
	if _, found := authDataMap[authConfigKey]; found && len(authConfigKey) > len(longestKey) {
		longestKey = authConfigKey
	}

	for registry, _ := range authDataMap {
		nameParts := strings.SplitN(registry, "/", 2)
		hostname := nameParts[0]

		// Only ever take a new key if the hostname matches in normal cases
		if authConfigKey == hostname {
			if longestKey == "" {
				longestKey = registry
			} else if len(registry) > len(longestKey) && strings.HasPrefix(image, registry) {
				// If we have a longer match, that indicates a username / namespace appended
				longestKey = registry
			}
		}
	}
	if longestKey != "" {
		return authDataMap[longestKey], nil
	}
	return docker.AuthConfiguration{}, nil
}
Esempio n. 17
0
// pushImage pushes a docker image to the registry specified in its tag
func pushImage(client DockerClient, name string, authConfig docker.AuthConfiguration) error {
	repository, tag := docker.ParseRepositoryTag(name)
	opts := docker.PushImageOptions{
		Name: repository,
		Tag:  tag,
	}
	if glog.V(5) {
		opts.OutputStream = os.Stderr
	}
	var err error
	for retries := 0; retries <= DefaultPushRetryCount; retries++ {
		err = client.PushImage(opts, authConfig)
		if err == nil {
			return nil
		}
		if retries == DefaultPushRetryCount {
			return err
		}
		util.HandleError(fmt.Errorf("push for image %s failed, will retry in %s ...", name, DefaultPushRetryDelay))
		glog.Flush()
		time.Sleep(DefaultPushRetryDelay)
	}
	return err
}
Esempio n. 18
0
// Looks up *docker.APIImage which matches *DockerImg passed as paramter and
// calls an anonymous func on it
func withDockerAPIImage(i *DockerImg, fn func(*docker.APIImages) error) error {
	return withDockerClient(func(client *docker.Client) error {
		imgs, err := client.ListImages(docker.ListImagesOptions{All: false})
		if err != nil {
			return err
		}
		for _, img := range imgs {
			for _, repoTag := range img.RepoTags {
				repo, tag := docker.ParseRepositoryTag(repoTag)
				if repo == i.Repo {
					if i.Tag == "" {
						if tag == "latest" {
							return fn(&img)
						}
					}
					if tag == i.Tag {
						return fn(&img)
					}
				}
			}
		}
		return fmt.Errorf("Image %s not found", i)
	})
}
Esempio n. 19
0
func (c *Client) pull(image string) ImageMetadata {
	reader, writer := io.Pipe()
	defer writer.Close()

	repository, tag := api.ParseRepositoryTag(image)
	tag = misc.NVL(tag, "latest")
	opts := api.PullImageOptions{
		Repository:   repository + ":" + tag,
		OutputStream: writer,
	}

	// check output goroutine
	began := make(chan bool, 1)
	once := sync.Once{}

	go func() {
		reader := bufio.NewReader(reader)
		var line string
		var err error
		for err == nil {
			line, err = reader.ReadString('\n')
			if err != nil {
				break
			}
			once.Do(func() {
				began <- true
			})
			if strings.Contains(line, "already being pulled by another client. Waiting.") {
				logs.Error.Printf("Image 'pull' status marked as already being pulled. image: %v, status: %v", opts.Repository, line)
			}
		}
		if err != nil && err != io.EOF {
			logs.Warn.Printf("Error reading pull image status. image: %v, err: %v", opts.Repository, err)
		}
	}()

	// pull the image
	timeout := time.After(cfg.DockerPullBeginTimeout)
	finished := make(chan error, 1)
	go func() {
		finished <- c.PullImage(opts, api.AuthConfiguration{})
		logs.Debug.Printf("Pull completed for image: %v", opts.Repository)
	}()

	// wait for the pulling to begin
	select {
	case <-began:
		break
	case err := <-finished:
		if err != nil {
			return ImageMetadata{
				Image: &api.Image{ID: ""},
				Error: CannotXContainerError{"Pull", err.Error()},
			}
		}
		return c.InspectImage(opts.Repository)
	case <-timeout:
		return ImageMetadata{
			Image: &api.Image{ID: ""},
			Error: &DockerTimeoutError{cfg.DockerPullBeginTimeout, "pullBegin"},
		}
	}

	// wait for the completion
	err := <-finished
	if err != nil {
		return ImageMetadata{
			Image: &api.Image{ID: ""},
			Error: CannotXContainerError{"Pull", err.Error()},
		}
	}
	return c.InspectImage(opts.Repository)
}
Esempio n. 20
0
func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
	// Get the image from config
	image, ok := task.Config["image"]
	if !ok || image == "" {
		return nil, fmt.Errorf("Image not specified")
	}
	if task.Resources == nil {
		return nil, fmt.Errorf("Resources are not specified")
	}
	if task.Resources.MemoryMB == 0 {
		return nil, fmt.Errorf("Memory limit cannot be zero")
	}
	if task.Resources.CPU == 0 {
		return nil, fmt.Errorf("CPU limit cannot be zero")
	}

	cleanupContainer, err := strconv.ParseBool(d.config.ReadDefault("docker.cleanup.container", "true"))
	if err != nil {
		return nil, fmt.Errorf("Unable to parse docker.cleanup.container: %s", err)
	}
	cleanupImage, err := strconv.ParseBool(d.config.ReadDefault("docker.cleanup.image", "true"))
	if err != nil {
		return nil, fmt.Errorf("Unable to parse docker.cleanup.image: %s", err)
	}

	// Initialize docker API client
	dockerEndpoint := d.config.ReadDefault("docker.endpoint", "unix:///var/run/docker.sock")
	client, err := docker.NewClient(dockerEndpoint)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to docker.endpoint (%s): %s", dockerEndpoint, err)
	}

	repo, tag := docker.ParseRepositoryTag(image)
	// Make sure tag is always explicitly set. We'll default to "latest" if it
	// isn't, which is the expected behavior.
	if tag == "" {
		tag = "latest"
	}

	var dockerImage *docker.Image
	// We're going to check whether the image is already downloaded. If the tag
	// is "latest" we have to check for a new version every time so we don't
	// bother to check and cache the id here. We'll download first, then cache.
	if tag != "latest" {
		dockerImage, err = client.InspectImage(image)
	}

	// Download the image
	if dockerImage == nil {
		pullOptions := docker.PullImageOptions{
			Repository: repo,
			Tag:        tag,
		}
		// TODO add auth configuration for private repos
		authOptions := docker.AuthConfiguration{}
		err = client.PullImage(pullOptions, authOptions)
		if err != nil {
			d.logger.Printf("[ERR] driver.docker: pulling container %s", err)
			return nil, fmt.Errorf("Failed to pull `%s`: %s", image, err)
		}
		d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded", repo, tag)

		// Now that we have the image we can get the image id
		dockerImage, err = client.InspectImage(image)
		if err != nil {
			d.logger.Printf("[ERR] driver.docker: getting image id for %s", image)
			return nil, fmt.Errorf("Failed to determine image id for `%s`: %s", image, err)
		}
	}
	d.logger.Printf("[DEBUG] driver.docker: using image %s", dockerImage.ID)
	d.logger.Printf("[INFO] driver.docker: identified image %s as %s", image, dockerImage.ID)

	// Create a container
	container, err := client.CreateContainer(createContainer(ctx, task, d.logger))
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: %s", err)
		return nil, fmt.Errorf("Failed to create container from image %s", image)
	}
	d.logger.Printf("[INFO] driver.docker: created container %s", container.ID)

	// Start the container
	err = client.StartContainer(container.ID, container.HostConfig)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: starting container %s", container.ID)
		return nil, fmt.Errorf("Failed to start container %s", container.ID)
	}
	d.logger.Printf("[INFO] driver.docker: started container %s", container.ID)

	// Return a driver handle
	h := &dockerHandle{
		client:           client,
		cleanupContainer: cleanupContainer,
		cleanupImage:     cleanupImage,
		logger:           d.logger,
		imageID:          dockerImage.ID,
		containerID:      container.ID,
		doneCh:           make(chan struct{}),
		waitCh:           make(chan error, 1),
	}
	go h.run()
	return h, nil
}
Esempio n. 21
0
func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
	var driverConfig DockerDriverConfig
	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
		return nil, err
	}
	image := driverConfig.ImageName

	if err := driverConfig.Validate(); err != nil {
		return nil, err
	}
	if task.Resources == nil {
		return nil, fmt.Errorf("Resources are not specified")
	}
	if task.Resources.MemoryMB == 0 {
		return nil, fmt.Errorf("Memory limit cannot be zero")
	}
	if task.Resources.CPU == 0 {
		return nil, fmt.Errorf("CPU limit cannot be zero")
	}

	cleanupContainer := d.config.ReadBoolDefault("docker.cleanup.container", true)
	cleanupImage := d.config.ReadBoolDefault("docker.cleanup.image", true)

	// Initialize docker API client
	client, err := d.dockerClient()
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
	}

	repo, tag := docker.ParseRepositoryTag(image)
	// Make sure tag is always explicitly set. We'll default to "latest" if it
	// isn't, which is the expected behavior.
	if tag == "" {
		tag = "latest"
	}

	var dockerImage *docker.Image
	// We're going to check whether the image is already downloaded. If the tag
	// is "latest" we have to check for a new version every time so we don't
	// bother to check and cache the id here. We'll download first, then cache.
	if tag != "latest" {
		dockerImage, err = client.InspectImage(image)
	}

	// Download the image
	if dockerImage == nil {
		pullOptions := docker.PullImageOptions{
			Repository: repo,
			Tag:        tag,
		}

		authOptions := docker.AuthConfiguration{}
		if len(driverConfig.Auth) != 0 {
			authOptions = docker.AuthConfiguration{
				Username:      driverConfig.Auth[0].Username,
				Password:      driverConfig.Auth[0].Password,
				Email:         driverConfig.Auth[0].Email,
				ServerAddress: driverConfig.Auth[0].ServerAddress,
			}
		}

		if authConfig := d.config.Read("docker.auth.config"); authConfig != "" {
			if f, err := os.Open(authConfig); err == nil {
				defer f.Close()
				if authConfigurations, err := docker.NewAuthConfigurations(f); err == nil {
					if authConfiguration, ok := authConfigurations.Configs[repo]; ok {
						authOptions = authConfiguration
					}
				}
			}
		}

		err = client.PullImage(pullOptions, authOptions)
		if err != nil {
			d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err)
			return nil, fmt.Errorf("Failed to pull `%s`: %s", image, err)
		}
		d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded", repo, tag)

		// Now that we have the image we can get the image id
		dockerImage, err = client.InspectImage(image)
		if err != nil {
			d.logger.Printf("[ERR] driver.docker: failed getting image id for %s: %s", image, err)
			return nil, fmt.Errorf("Failed to determine image id for `%s`: %s", image, err)
		}
	}

	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
	if !ok {
		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
	}

	d.logger.Printf("[DEBUG] driver.docker: identified image %s as %s", image, dockerImage.ID)

	bin, err := discover.NomadExecutable()
	if err != nil {
		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
	}
	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-syslog-collector.out", task.Name))
	pluginConfig := &plugin.ClientConfig{
		Cmd: exec.Command(bin, "syslog", pluginLogFile),
	}

	logCollector, pluginClient, err := createLogCollector(pluginConfig, d.config.LogOutput, d.config)
	if err != nil {
		return nil, err
	}
	logCollectorCtx := &logcollector.LogCollectorContext{
		TaskName:       task.Name,
		AllocDir:       ctx.AllocDir,
		LogConfig:      task.LogConfig,
		PortLowerBound: d.config.ClientMinPort,
		PortUpperBound: d.config.ClientMaxPort,
	}
	ss, err := logCollector.LaunchCollector(logCollectorCtx)
	if err != nil {
		return nil, fmt.Errorf("failed to start syslog collector: %v", err)
	}

	config, err := d.createContainer(ctx, task, &driverConfig, ss.Addr)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %s: %s", image, err)
		pluginClient.Kill()
		return nil, fmt.Errorf("Failed to create container configuration for image %s: %s", image, err)
	}
	// Create a container
	container, err := client.CreateContainer(config)
	if err != nil {
		// If the container already exists because of a previous failure we'll
		// try to purge it and re-create it.
		if strings.Contains(err.Error(), "container already exists") {
			// Get the ID of the existing container so we can delete it
			containers, err := client.ListContainers(docker.ListContainersOptions{
				// The image might be in use by a stopped container, so check everything
				All: true,
				Filters: map[string][]string{
					"name": []string{config.Name},
				},
			})
			if err != nil {
				d.logger.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name)
				pluginClient.Kill()
				return nil, fmt.Errorf("Failed to query list of containers: %s", err)
			}

			// Couldn't find any matching containers
			if len(containers) == 0 {
				d.logger.Printf("[ERR] driver.docker: failed to get id for container %s: %#v", config.Name, containers)
				pluginClient.Kill()
				return nil, fmt.Errorf("Failed to get id for container %s", config.Name)
			}

			// Delete matching containers
			d.logger.Printf("[INFO] driver.docker: a container with the name %s already exists; will attempt to purge and re-create", config.Name)
			for _, container := range containers {
				err = client.RemoveContainer(docker.RemoveContainerOptions{
					ID: container.ID,
				})
				if err != nil {
					d.logger.Printf("[ERR] driver.docker: failed to purge container %s", container.ID)
					pluginClient.Kill()
					return nil, fmt.Errorf("Failed to purge container %s: %s", container.ID, err)
				}
				d.logger.Printf("[INFO] driver.docker: purged container %s", container.ID)
			}

			container, err = client.CreateContainer(config)
			if err != nil {
				d.logger.Printf("[ERR] driver.docker: failed to re-create container %s; aborting", config.Name)
				pluginClient.Kill()
				return nil, fmt.Errorf("Failed to re-create container %s; aborting", config.Name)
			}
		} else {
			// We failed to create the container for some other reason.
			d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s", image, err)
			pluginClient.Kill()
			return nil, fmt.Errorf("Failed to create container from image %s: %s", image, err)
		}
	}
	d.logger.Printf("[INFO] driver.docker: created container %s", container.ID)

	// Start the container
	err = client.StartContainer(container.ID, container.HostConfig)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err)
		pluginClient.Kill()
		return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err)
	}
	d.logger.Printf("[INFO] driver.docker: started container %s", container.ID)

	// Return a driver handle
	h := &DockerHandle{
		client:           client,
		logCollector:     logCollector,
		pluginClient:     pluginClient,
		cleanupContainer: cleanupContainer,
		cleanupImage:     cleanupImage,
		logger:           d.logger,
		imageID:          dockerImage.ID,
		containerID:      container.ID,
		killTimeout:      d.DriverContext.KillTimeout(task),
		doneCh:           make(chan struct{}),
		waitCh:           make(chan *cstructs.WaitResult, 1),
	}
	go h.run()
	return h, nil
}
func (dg *dockerGoClient) pullImage(image string, authData *api.RegistryAuthenticationData) DockerContainerMetadata {
	log.Debug("Pulling image", "image", image)
	client, err := dg.dockerClient()
	if err != nil {
		return DockerContainerMetadata{Error: CannotGetDockerClientError{version: dg.version, err: err}}
	}

	// Special case; this image is not one that should be pulled, but rather
	// should be created locally if necessary
	if image == emptyvolume.Image+":"+emptyvolume.Tag {
		scratchErr := dg.createScratchImageIfNotExists()
		if scratchErr != nil {
			return DockerContainerMetadata{Error: &api.DefaultNamedError{Name: "CreateEmptyVolumeError", Err: "Could not create empty volume " + scratchErr.Error()}}
		}
		return DockerContainerMetadata{}
	}

	authConfig, err := dg.getAuthdata(image, authData)
	if err != nil {
		return DockerContainerMetadata{Error: CannotXContainerError{"Pull", err.Error()}}
	}

	pullDebugOut, pullWriter := io.Pipe()
	defer pullWriter.Close()

	repository, tag := docker.ParseRepositoryTag(image)
	if tag == "" {
		repository = repository + ":" + dockerDefaultTag
	} else {
		repository = image
	}

	opts := docker.PullImageOptions{
		Repository:   repository,
		OutputStream: pullWriter,
	}
	timeout := dg.time().After(dockerPullBeginTimeout)
	// pullBegan is a channel indicating that we have seen at least one line of data on the 'OutputStream' above.
	// It is here to guard against a bug wherin docker never writes anything to that channel and hangs in pulling forever.
	pullBegan := make(chan bool, 1)
	// pullBeganOnce ensures we only indicate it began once (since our channel will only be read 0 or 1 times)
	pullBeganOnce := sync.Once{}

	go func() {
		reader := bufio.NewReader(pullDebugOut)
		var line string
		var pullErr error
		var statusDisplayed time.Time
		for pullErr == nil {
			line, pullErr = reader.ReadString('\n')
			if pullErr != nil {
				break
			}
			pullBeganOnce.Do(func() {
				pullBegan <- true
			})

			now := time.Now()
			if !strings.Contains(line, "[=") || now.After(statusDisplayed.Add(pullStatusSuppressDelay)) {
				// skip most of the progress bar lines, but retain enough for debugging
				log.Debug("Pulling image", "image", image, "status", line)
				statusDisplayed = now
			}

			if strings.Contains(line, "already being pulled by another client. Waiting.") {
				// This can mean the daemon is 'hung' in pulling status for this image, but we can't be sure.
				log.Error("Image 'pull' status marked as already being pulled", "image", image, "status", line)
			}
		}
		if pullErr != nil && pullErr != io.EOF {
			log.Warn("Error reading pull image status", "image", image, "err", pullErr)
		}
	}()
	pullFinished := make(chan error, 1)
	go func() {
		pullFinished <- client.PullImage(opts, authConfig)
		log.Debug("Pulling image complete", "image", image)
	}()

	select {
	case <-pullBegan:
		break
	case pullErr := <-pullFinished:
		if pullErr != nil {
			return DockerContainerMetadata{Error: CannotXContainerError{"Pull", pullErr.Error()}}
		}
		return DockerContainerMetadata{}
	case <-timeout:
		return DockerContainerMetadata{Error: &DockerTimeoutError{dockerPullBeginTimeout, "pullBegin"}}
	}
	log.Debug("Pull began for image", "image", image)
	defer log.Debug("Pull completed for image", "image", image)

	err = <-pullFinished
	if err != nil {
		return DockerContainerMetadata{Error: CannotXContainerError{"Pull", err.Error()}}
	}
	return DockerContainerMetadata{}
}
Esempio n. 23
0
func init() {
	cfg := config.NewConfig()

	http.Handle("/container/top/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		id := r.URL.Path[len("/container/top/"):]
		client, _ := util.RequestGetParam(r, "client")
		params := struct{ ID, Name, Client string }{id, _label(id, client, cfg.LabelOverrideNames), client}
		util.RenderHTML(w, []string{"containers/top.tmpl"}, params, nil)
	}))
	http.Handle("/container/statlog/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		id := r.URL.Path[len("/container/statlog/"):]
		client, _ := util.RequestGetParam(r, "client")
		params := struct {
			ID, Name, Client string
			ViewOnly         bool
		}{id, _label(id, client, cfg.LabelOverrideNames), client, cfg.ViewOnly}
		util.RenderHTML(w, []string{"containers/statlog.tmpl"}, params, nil)
	}))
	http.Handle("/container/changes/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		id := r.URL.Path[len("/container/changes/"):]
		client, _ := util.RequestGetParam(r, "client")
		params := struct{ ID, Name, Client string }{id, _label(id, client, cfg.LabelOverrideNames), client}
		util.RenderHTML(w, []string{"containers/changes.tmpl"}, params, nil)
	}))
	http.Handle("/logs", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		params := struct{ LabelFilters string }{strings.Join(cfg.LabelFilters, ",")}
		util.RenderHTML(w, []string{"containers/logs.tmpl"}, params, nil)
	}))
	http.Handle("/statistics", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		clients, err := models.LoadDockerClients()
		if err != nil {
			renderErrorJSON(w, err)
			return
		}
		params := struct {
			LabelFilters string
			Clients      int
		}{strings.Join(cfg.LabelFilters, ","), len(clients)}
		util.RenderHTML(w, []string{"containers/statistics.tmpl"}, params, nil)
	}))

	/**
	 * Containers' API
	 * @param limit int
	 * @param status int (0: all, 1: created, 2: restarting, 3: running, 4: paused, 5&6: exited)
	 * @param q string search words
	 * @return []model.DockerContainer
	 */
	http.Handle("/api/containers", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		type container struct {
			Client     *models.DockerClient     `json:"client"`
			Containers []models.DockerContainer `json:"containers"`
		}
		dockers, ok := clients(w)
		if !ok {
			return
		}
		options := models.ListContainerOption(util.RequestGetParamI(r, "status", 0))
		options.Limit = util.RequestGetParamI(r, "limit", 100)
		d := make(chan *container, len(dockers))
		result := []*container{}

		for _, docker := range dockers {
			go func(docker *engine.Client) {
				containers, err := docker.ListContainers(options)
				if err != nil {
					renderErrorJSON(w, err)
					return
				}
				var words []string
				if q, found := util.RequestGetParam(r, "q"); found {
					words = util.SplittedUpperStrings(q)
				}
				d <- &container{
					Client:     docker.Conf,
					Containers: models.SearchContainers(containers, words),
				}
			}(docker)
		}
		for i := 0; i < len(dockers); i++ {
			result = append(result, <-d)
		}
		close(d)
		util.RenderJSON(w, result, nil)
	}))

	http.Handle("/api/statistics", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		var dockers []*engine.Client
		if c, found := util.RequestGetParam(r, "client"); found {
			if docker, ok := client(w, c); ok {
				dockers = []*engine.Client{docker}
			}
		} else {
			var ok bool
			dockers, ok = clients(w)
			if !ok {
				return
			}
		}
		type statistics struct {
			Client *models.DockerClient               `json:"client"`
			Stats  map[string]map[string][]*api.Stats `json:"stats"`
		}
		stats := []statistics{}

		d := make(chan statistics, len(dockers))
		for _, docker := range dockers {
			go func(docker *engine.Client) {
				candidate, err := docker.ListContainers(models.ListContainerOption(3))
				if err != nil {
					renderErrorJSON(w, err)
					return
				}
				containers := models.SearchContainers(candidate, []string{})
				c := make(chan models.DockerStats, len(containers))
				stats := map[string]map[string][]*api.Stats{}
				count := util.RequestGetParamI(r, "count", 1)

				for _, container := range containers {
					go func(container models.DockerContainer) {
						stat, _ := docker.Stats(container.ID, count)

						name := strings.Join(container.Names, ",")
						if !misc.ZeroOrNil(cfg.LabelOverrideNames) {
							if label, found := container.Labels[cfg.LabelOverrideNames]; found {
								name = "*" + label
							}
						}
						c <- models.DockerStats{
							ID:    container.ID,
							Name:  name,
							Stats: stat,
						}
					}(container)
				}
				for i := 0; i < len(containers); i++ {
					ds := <-c
					inner := map[string][]*api.Stats{}
					inner[ds.Name] = ds.Stats
					stats[ds.ID] = inner
				}
				close(c)
				d <- statistics{
					Client: docker.Conf,
					Stats:  stats,
				}
			}(docker)
		}
		for i := 0; i < len(dockers); i++ {
			stats = append(stats, <-d)
		}
		close(d)
		util.RenderJSON(w, stats, nil)
	}))

	http.Handle("/api/logs", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		count := util.RequestGetParamI(r, "count", 100)
		var dockers []*engine.Client
		if c, found := util.RequestGetParam(r, "client"); found {
			if docker, ok := client(w, c); ok {
				dockers = []*engine.Client{docker}
			}
		} else {
			var ok bool
			dockers, ok = clients(w)
			if !ok {
				return
			}
		}
		type stdlogs struct {
			ID     string   `json:"id"`
			Stdout []string `json:"stdout"`
			Stderr []string `json:"stderr"`
		}
		type clientlogs struct {
			Client *models.DockerClient `json:"client"`
			Logs   []stdlogs            `json:"logs"`
		}
		logs := []clientlogs{}

		d := make(chan clientlogs, len(dockers))
		for _, docker := range dockers {
			go func(docker *engine.Client) {
				candidate, err := docker.ListContainers(models.ListContainerOption(3))
				if err != nil {
					renderErrorJSON(w, err)
					return
				}
				containers := models.SearchContainers(candidate, []string{})

				c := make(chan stdlogs, len(containers))
				inner := []stdlogs{}
				for _, container := range containers {
					go func(container models.DockerContainer) {

						stdout, stderr, err := docker.Logs(container.ID, count, 1*time.Second)
						if err != nil {
							renderErrorJSON(w, err)
							return
						}
						c <- stdlogs{container.ID, stdout, stderr}
					}(container)
				}
				for i := 0; i < len(containers); i++ {
					inner = append(inner, <-c)
				}
				close(c)
				d <- clientlogs{
					Client: docker.Conf,
					Logs:   inner,
				}
			}(docker)
		}
		for i := 0; i < len(dockers); i++ {
			logs = append(logs, <-d)
		}
		close(d)
		util.RenderJSON(w, logs, nil)
	}))

	/**
	 * A container's API
	 */
	// inspect
	http.Handle("/api/container/inspect/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if docker, ok := client(w, util.RequestGetParamS(r, "client", "")); ok {
			id := r.URL.Path[len("/api/container/inspect/"):]
			meta := docker.InspectContainer(id)
			util.RenderJSON(w, meta.Container, meta.Error)
		}
	}))
	// top
	http.Handle("/api/container/top/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if docker, ok := client(w, util.RequestGetParamS(r, "client", "")); ok {
			id := r.URL.Path[len("/api/container/top/"):]
			args := util.RequestGetParamS(r, "args", "aux")
			util.RenderJSON(w, docker.Top(id, args), nil)
		}
	}))
	// stats
	http.Handle("/api/container/stats/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if docker, ok := client(w, util.RequestGetParamS(r, "client", "")); ok {
			id := r.URL.Path[len("/api/container/stats/"):]
			result, err := docker.Stats(id, util.RequestGetParamI(r, "count", 1))
			if err != nil {
				renderErrorJSON(w, err)
				return
			}
			util.RenderJSON(w, result, nil)
		}
	}))
	// logs
	http.Handle("/api/container/logs/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if docker, ok := client(w, util.RequestGetParamS(r, "client", "")); ok {
			id := r.URL.Path[len("/api/container/logs/"):]
			count := util.RequestGetParamI(r, "count", 100)
			stdout, stderr, err := docker.Logs(id, count, 1*time.Second)
			if err != nil {
				renderErrorJSON(w, err)
				return
			}
			util.RenderJSON(w, struct {
				Stdout []string `json:"stdout"`
				Stderr []string `json:"stderr"`
			}{
				stdout,
				stderr,
			}, nil)
		}
	}))
	// diff
	http.Handle("/api/container/changes/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if docker, ok := client(w, util.RequestGetParamS(r, "client", "")); ok {
			id := r.URL.Path[len("/api/container/changes/"):]
			util.RenderJSON(w, docker.Changes(id), nil)
		}
	}))

	// restart
	http.Handle("/api/container/restart/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.NotFound(w, r)
			return
		}
		if docker, ok := client(w, util.RequestPostParamS(r, "client", "")); ok {
			meta := docker.Restart(r.URL.Path[len("/api/container/restart/"):], 5)
			if meta.Error != nil {
				renderErrorJSON(w, meta.Error)
				return
			}
			util.RenderJSON(w, meta.Container, nil)
		}
	}))
	// start
	http.Handle("/api/container/start/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.NotFound(w, r)
			return
		}
		if docker, ok := client(w, util.RequestPostParamS(r, "client", "")); ok {
			meta := docker.Start(r.URL.Path[len("/api/container/start/"):])
			if meta.Error != nil {
				renderErrorJSON(w, meta.Error)
				return
			}
			util.RenderJSON(w, meta.Container, nil)
		}
	}))
	// stop
	http.Handle("/api/container/stop/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.NotFound(w, r)
			return
		}
		if docker, ok := client(w, util.RequestPostParamS(r, "client", "")); ok {
			meta := docker.Stop(r.URL.Path[len("/api/container/stop/"):])
			if meta.Error != nil {
				renderErrorJSON(w, meta.Error)
				return
			}
			util.RenderJSON(w, meta.Container, nil)
		}
	}))
	// kill
	http.Handle("/api/container/kill/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.NotFound(w, r)
			return
		}
		if docker, ok := client(w, util.RequestPostParamS(r, "client", "")); ok {
			meta := docker.Kill(r.URL.Path[len("/api/container/kill/"):], 5)
			if meta.Error != nil {
				renderErrorJSON(w, meta.Error)
				return
			}
			util.RenderJSON(w, meta.Container, nil)
		}
	}))
	// rm
	http.Handle("/api/container/rm/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.NotFound(w, r)
			return
		}
		if docker, ok := client(w, util.RequestPostParamS(r, "client", "")); ok {
			err := docker.Rm(r.URL.Path[len("/api/container/rm/"):])
			if err != nil {
				renderErrorJSON(w, err)
				return
			}
			util.RenderJSON(w, "removed successfully.", nil)
		}
	}))
	// rename
	http.Handle("/api/container/rename/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.NotFound(w, r)
			return
		}
		if docker, ok := client(w, util.RequestPostParamS(r, "client", "")); ok {
			if name, found := util.RequestPostParam(r, "name"); found {
				err := docker.Rename(r.URL.Path[len("/api/container/rename/"):], name)
				message := "renamed successfully."
				if err != nil {
					message = err.Error()
				}
				util.RenderJSON(w, message, nil)
				return
			}
		}
	}))
	// commit
	http.Handle("/api/container/commit/", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.NotFound(w, r)
			return
		}
		if docker, ok := client(w, util.RequestPostParamS(r, "client", "")); ok {
			repository, _ := util.RequestPostParam(r, "repo")
			tag, _ := util.RequestPostParam(r, "tag")
			massage, _ := util.RequestPostParam(r, "msg")
			author, _ := util.RequestPostParam(r, "author")

			meta := docker.Commit(
				r.URL.Path[len("/api/container/commit/"):],
				repository, tag, massage, author)
			if meta.Error != nil {
				renderErrorJSON(w, meta.Error)
				return
			}
			util.RenderJSON(w, meta.Image, nil)
		}
	}))

	// @see https://docs.docker.com/docker-hub/builds/#webhooks
	type Repository struct {
		Name     string `json:"name"`
		Owner    string `json:"owner"`
		RepoName string `json:"repo_name"`
	}
	type Webhook struct {
		CallbackURL string     `json:"callback_url"`
		Repository  Repository `json:"repository"`
	}

	/**
	 * Update by DockerHub
	 * which pull an image again & restart the container to update its service
	 */
	http.Handle("/api/container/update", util.Chain(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.NotFound(w, r)
			return
		}
		// parse parameters
		webhook := Webhook{}
		err := json.NewDecoder(r.Body).Decode(&webhook)
		if err != nil {
			logs.Warn.Print(err)
			http.NotFound(w, r)
			return
		}
		repository, tag := api.ParseRepositoryTag(webhook.Repository.RepoName)
		tag = misc.NVL(tag, "latest")

		// TODO trying all docker clients!!
		docker, ok := client(w, util.RequestPostParamS(r, "client", ""))
		if !ok {
			return
		}

		// pull the latest image
		var imageRepoTag string
		for _, image := range docker.ListImages() {
			for _, repotag := range image.RepoTags {
				if repotag == repository+":"+tag {
					imageRepoTag = repotag
				}
			}
		}
		if misc.ZeroOrNil(imageRepoTag) {
			http.NotFound(w, r)
			return
		}
		if meta := docker.Pull(imageRepoTag); meta.Error != nil {
			util.RenderJSON(w, meta.Error.Error(), nil)
			return
		}

		// list running containers
		containers, err := docker.ListContainers(models.ListContainerOption(3))
		if err != nil {
			util.RenderJSON(w, err.Error(), nil)
			return
		}

		// restart its container
		restarted := []string{}
		for _, container := range containers {
			if container.Image == imageRepoTag {
				meta := docker.InspectContainer(container.ID)
				if meta.Error != nil {
					continue
				}
				// remove the existing container
				if meta := docker.Stop(container.ID); meta.Error != nil {
					renderErrorJSON(w, meta.Error)
					return
				}
				if err := docker.Rm(container.ID); err != nil {
					renderErrorJSON(w, err)
					return
				}
				// create a new container using the dead container configurations.
				// because if we just restart the container, its image would not
				// reference the new one.
				c := meta.Container
				if meta := docker.Create(c.Name, c.Config, c.HostConfig); meta.Error != nil {
					renderErrorJSON(w, meta.Error)
					return
				}
				if meta := docker.Start(c.Name[1:]); meta.Error != nil {
					renderErrorJSON(w, meta.Error)
					return
				}
				restarted = append(restarted, c.Name[1:])
			}
		}
		util.RenderJSON(w, restarted, nil)
	}))
}
Esempio n. 24
0
func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
	var driverConfig DockerDriverConfig
	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
		return nil, err
	}
	image := driverConfig.ImageName

	if err := driverConfig.Validate(); err != nil {
		return nil, err
	}
	if task.Resources == nil {
		return nil, fmt.Errorf("Resources are not specified")
	}
	if task.Resources.MemoryMB == 0 {
		return nil, fmt.Errorf("Memory limit cannot be zero")
	}
	if task.Resources.CPU == 0 {
		return nil, fmt.Errorf("CPU limit cannot be zero")
	}

	cleanupContainer := d.config.ReadBoolDefault("docker.cleanup.container", true)
	cleanupImage := d.config.ReadBoolDefault("docker.cleanup.image", true)

	// Initialize docker API client
	client, err := d.dockerClient()
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
	}

	repo, tag := docker.ParseRepositoryTag(image)
	// Make sure tag is always explicitly set. We'll default to "latest" if it
	// isn't, which is the expected behavior.
	if tag == "" {
		tag = "latest"
	}

	var dockerImage *docker.Image
	// We're going to check whether the image is already downloaded. If the tag
	// is "latest" we have to check for a new version every time so we don't
	// bother to check and cache the id here. We'll download first, then cache.
	if tag != "latest" {
		dockerImage, err = client.InspectImage(image)
	}

	// Download the image
	if dockerImage == nil {
		pullOptions := docker.PullImageOptions{
			Repository: repo,
			Tag:        tag,
		}

		authOptions := docker.AuthConfiguration{}
		if len(driverConfig.Auth) != 0 {
			authOptions = docker.AuthConfiguration{
				Username:      driverConfig.Auth[0].Username,
				Password:      driverConfig.Auth[0].Password,
				Email:         driverConfig.Auth[0].Email,
				ServerAddress: driverConfig.Auth[0].ServerAddress,
			}
		}

		err = client.PullImage(pullOptions, authOptions)
		if err != nil {
			d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err)
			return nil, fmt.Errorf("Failed to pull `%s`: %s", image, err)
		}
		d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded", repo, tag)

		// Now that we have the image we can get the image id
		dockerImage, err = client.InspectImage(image)
		if err != nil {
			d.logger.Printf("[ERR] driver.docker: failed getting image id for %s: %s", image, err)
			return nil, fmt.Errorf("Failed to determine image id for `%s`: %s", image, err)
		}
	}
	d.logger.Printf("[DEBUG] driver.docker: identified image %s as %s", image, dockerImage.ID)

	config, err := d.createContainer(ctx, task, &driverConfig)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %s: %s", image, err)
		return nil, fmt.Errorf("Failed to create container configuration for image %s: %s", image, err)
	}
	// Create a container
	container, err := client.CreateContainer(config)
	if err != nil {
		// If the container already exists because of a previous failure we'll
		// try to purge it and re-create it.
		if strings.Contains(err.Error(), "container already exists") {
			// Get the ID of the existing container so we can delete it
			containers, err := client.ListContainers(docker.ListContainersOptions{
				// The image might be in use by a stopped container, so check everything
				All: true,
				Filters: map[string][]string{
					"name": []string{config.Name},
				},
			})
			if err != nil {
				log.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name)
				return nil, fmt.Errorf("Failed to query list of containers: %s", err)
			}

			if len(containers) != 1 {
				log.Printf("[ERR] driver.docker: failed to get id for container %s", config.Name)
				return nil, fmt.Errorf("Failed to get id for container %s", config.Name)
			}

			log.Printf("[INFO] driver.docker: a container with the name %s already exists; will attempt to purge and re-create", config.Name)
			err = client.RemoveContainer(docker.RemoveContainerOptions{
				ID: containers[0].ID,
			})
			if err != nil {
				log.Printf("[ERR] driver.docker: failed to purge container %s", config.Name)
				return nil, fmt.Errorf("Failed to purge container %s: %s", config.Name, err)
			}
			log.Printf("[INFO] driver.docker: purged container %s", config.Name)
			container, err = client.CreateContainer(config)
			if err != nil {
				log.Printf("[ERR] driver.docker: failed to re-create container %s; aborting", config.Name)
				return nil, fmt.Errorf("Failed to re-create container %s; aborting", config.Name)
			}
		} else {
			// We failed to create the container for some other reason.
			d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s", image, err)
			return nil, fmt.Errorf("Failed to create container from image %s: %s", image, err)
		}
	}
	d.logger.Printf("[INFO] driver.docker: created container %s", container.ID)

	// Start the container
	err = client.StartContainer(container.ID, container.HostConfig)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err)
		return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err)
	}
	d.logger.Printf("[INFO] driver.docker: started container %s", container.ID)

	// Return a driver handle
	h := &DockerHandle{
		client:           client,
		cleanupContainer: cleanupContainer,
		cleanupImage:     cleanupImage,
		logger:           d.logger,
		imageID:          dockerImage.ID,
		containerID:      container.ID,
		doneCh:           make(chan struct{}),
		waitCh:           make(chan *cstructs.WaitResult, 1),
	}
	go h.run()
	return h, nil
}
Esempio n. 25
0
func ImageName(opts *dc.PullImageOptions, image string) {
	opts.Repository, opts.Tag = dc.ParseRepositoryTag(image)
}
Esempio n. 26
0
// Build is a helper method to perform a Docker build against the
// provided Docker client. It will load the image if not specified,
// create a container if one does not already exist, and start a
// container if the Dockerfile contains RUN commands. It will cleanup
// any containers it creates directly, and set the e.Image.ID field
// to the generated image.
func (e *ClientExecutor) Build(r io.Reader, args map[string]string) error {
	b := NewBuilder()
	b.Args = args

	if e.Excludes == nil {
		excludes, err := ParseDockerignore(e.Directory)
		if err != nil {
			return err
		}
		e.Excludes = append(excludes, ".dockerignore")
	}

	// TODO: check the Docker daemon version (1.20 is required for Upload)

	node, err := parser.Parse(r)
	if err != nil {
		return err
	}

	// identify the base image
	from, err := b.From(node)
	if err != nil {
		return err
	}
	// load the image
	if e.Image == nil {
		if from == NoBaseImageSpecifier {
			if runtime.GOOS == "windows" {
				return fmt.Errorf("building from scratch images is not supported")
			}
			from, err = e.CreateScratchImage()
			if err != nil {
				return err
			}
			defer e.CleanupImage(from)
		}
		glog.V(4).Infof("Retrieving image %q", from)
		e.Image, err = e.LoadImage(from)
		if err != nil {
			return err
		}
	}

	// update the builder with any information from the image, including ONBUILD
	// statements
	if err := b.FromImage(e.Image, node); err != nil {
		return err
	}

	b.RunConfig.Image = from
	e.LogFn("FROM %s", from)
	glog.V(4).Infof("step: FROM %s", from)

	// create a container to execute in, if necessary
	mustStart := b.RequiresStart(node)
	if e.Container == nil {
		opts := docker.CreateContainerOptions{
			Config: &docker.Config{
				Image: from,
			},
		}
		if mustStart {
			// TODO: windows support
			if len(e.Command) > 0 {
				opts.Config.Cmd = e.Command
				opts.Config.Entrypoint = nil
			} else {
				// TODO; replace me with a better default command
				opts.Config.Cmd = []string{"sleep 86400"}
				opts.Config.Entrypoint = []string{"/bin/sh", "-c"}
			}
		}
		if len(opts.Config.Cmd) == 0 {
			opts.Config.Entrypoint = []string{"/bin/sh", "-c", "# NOP"}
		}
		container, err := e.Client.CreateContainer(opts)
		if err != nil {
			return err
		}
		e.Container = container

		// if we create the container, take responsibilty for cleaning up
		defer e.Cleanup()
	}

	// TODO: lazy start
	if mustStart && !e.Container.State.Running {
		if err := e.Client.StartContainer(e.Container.ID, e.HostConfig); err != nil {
			return err
		}
		// TODO: is this racy? may have to loop wait in the actual run step
	}

	for _, child := range node.Children {
		step := b.Step()
		if err := step.Resolve(child); err != nil {
			return err
		}
		glog.V(4).Infof("step: %s", step.Original)
		if e.LogFn != nil {
			e.LogFn(step.Original)
		}
		if err := b.Run(step, e); err != nil {
			return err
		}
	}

	if mustStart {
		glog.V(4).Infof("Stopping container %s ...", e.Container.ID)
		if err := e.Client.StopContainer(e.Container.ID, 0); err != nil {
			return err
		}
	}

	config := b.Config()
	var repository, tag string
	if len(e.Tag) > 0 {
		repository, tag = docker.ParseRepositoryTag(e.Tag)
		glog.V(4).Infof("Committing built container %s as image %q: %#v", e.Container.ID, e.Tag, config)
		if e.LogFn != nil {
			e.LogFn("Committing changes to %s ...", e.Tag)
		}
	} else {
		glog.V(4).Infof("Committing built container %s: %#v", e.Container.ID, config)
		if e.LogFn != nil {
			e.LogFn("Committing changes ...")
		}
	}

	image, err := e.Client.CommitContainer(docker.CommitContainerOptions{
		Author:     b.Author,
		Container:  e.Container.ID,
		Run:        config,
		Repository: repository,
		Tag:        tag,
	})
	if err != nil {
		return err
	}
	e.Image = image
	glog.V(4).Infof("Committed %s to %s", e.Container.ID, e.Image.ID)
	if e.LogFn != nil {
		e.LogFn("Done")
	}
	return nil
}
Esempio n. 27
0
// Build is a helper method to perform a Docker build against the
// provided Docker client. It will load the image if not specified,
// create a container if one does not already exist, and start a
// container if the Dockerfile contains RUN commands. It will cleanup
// any containers it creates directly, and set the e.Image.ID field
// to the generated image.
func (e *ClientExecutor) Build(r io.Reader, args map[string]string) error {
	b := NewBuilder()
	b.Args = args

	if e.Excludes == nil {
		excludes, err := ParseDockerignore(e.Directory)
		if err != nil {
			return err
		}
		e.Excludes = append(excludes, ".dockerignore")
	}

	// TODO: check the Docker daemon version (1.20 is required for Upload)

	node, err := parser.Parse(r)
	if err != nil {
		return err
	}

	// identify the base image
	from, err := b.From(node)
	if err != nil {
		return err
	}
	// load the image
	if e.Image == nil {
		if from == NoBaseImageSpecifier {
			if runtime.GOOS == "windows" {
				return fmt.Errorf("building from scratch images is not supported")
			}
			from, err = e.CreateScratchImage()
			if err != nil {
				return fmt.Errorf("unable to create a scratch image for this build: %v", err)
			}
			defer e.CleanupImage(from)
		}
		glog.V(4).Infof("Retrieving image %q", from)
		e.Image, err = e.LoadImage(from)
		if err != nil {
			return err
		}
	}

	// update the builder with any information from the image, including ONBUILD
	// statements
	if err := b.FromImage(e.Image, node); err != nil {
		return err
	}

	b.RunConfig.Image = from
	e.LogFn("FROM %s", from)
	glog.V(4).Infof("step: FROM %s", from)

	var sharedMount string

	// create a container to execute in, if necessary
	mustStart := b.RequiresStart(node)
	if e.Container == nil {
		opts := docker.CreateContainerOptions{
			Config: &docker.Config{
				Image: from,
			},
		}
		if mustStart {
			// Transient mounts only make sense on images that will be running processes
			if len(e.TransientMounts) > 0 {
				volumeName, err := randSeq(imageSafeCharacters, 24)
				if err != nil {
					return err
				}
				v, err := e.Client.CreateVolume(docker.CreateVolumeOptions{Name: volumeName})
				if err != nil {
					return fmt.Errorf("unable to create volume to mount secrets: %v", err)
				}
				defer e.cleanupVolume(volumeName)
				sharedMount = v.Mountpoint
				opts.HostConfig = &docker.HostConfig{
					Binds: []string{sharedMount + ":/tmp/__temporarymount"},
				}
			}

			// TODO: windows support
			if len(e.Command) > 0 {
				opts.Config.Cmd = e.Command
				opts.Config.Entrypoint = nil
			} else {
				// TODO; replace me with a better default command
				opts.Config.Cmd = []string{"sleep 86400"}
				opts.Config.Entrypoint = []string{"/bin/sh", "-c"}
			}
		}
		if len(opts.Config.Cmd) == 0 {
			opts.Config.Entrypoint = []string{"/bin/sh", "-c", "# NOP"}
		}
		container, err := e.Client.CreateContainer(opts)
		if err != nil {
			return fmt.Errorf("unable to create build container: %v", err)
		}
		e.Container = container

		// if we create the container, take responsibilty for cleaning up
		defer e.Cleanup()
	}

	// copy any source content into the temporary mount path
	if mustStart && len(e.TransientMounts) > 0 {
		var copies []Copy
		for i, mount := range e.TransientMounts {
			source := mount.SourcePath
			copies = append(copies, Copy{
				Src:  source,
				Dest: []string{path.Join("/tmp/__temporarymount", strconv.Itoa(i))},
			})
		}
		if err := e.Copy(copies...); err != nil {
			return fmt.Errorf("unable to copy build context into container: %v", err)
		}
	}

	// TODO: lazy start
	if mustStart && !e.Container.State.Running {
		var hostConfig docker.HostConfig
		if e.HostConfig != nil {
			hostConfig = *e.HostConfig
		}

		// mount individual items temporarily
		for i, mount := range e.TransientMounts {
			if len(sharedMount) == 0 {
				return fmt.Errorf("no mount point available for temporary mounts")
			}
			hostConfig.Binds = append(
				hostConfig.Binds,
				fmt.Sprintf("%s:%s:%s", path.Join(sharedMount, strconv.Itoa(i)), mount.DestinationPath, "ro"),
			)
		}

		if err := e.Client.StartContainer(e.Container.ID, &hostConfig); err != nil {
			return fmt.Errorf("unable to start build container: %v", err)
		}
		// TODO: is this racy? may have to loop wait in the actual run step
	}

	for _, child := range node.Children {
		step := b.Step()
		if err := step.Resolve(child); err != nil {
			return err
		}
		glog.V(4).Infof("step: %s", step.Original)
		if e.LogFn != nil {
			e.LogFn(step.Original)
		}
		if err := b.Run(step, e); err != nil {
			return err
		}
	}

	if mustStart {
		glog.V(4).Infof("Stopping container %s ...", e.Container.ID)
		if err := e.Client.StopContainer(e.Container.ID, 0); err != nil {
			return fmt.Errorf("unable to stop build container: %v", err)
		}
	}

	config := b.Config()
	var repository, tag string
	if len(e.Tag) > 0 {
		repository, tag = docker.ParseRepositoryTag(e.Tag)
		glog.V(4).Infof("Committing built container %s as image %q: %#v", e.Container.ID, e.Tag, config)
		if e.LogFn != nil {
			e.LogFn("Committing changes to %s ...", e.Tag)
		}
	} else {
		glog.V(4).Infof("Committing built container %s: %#v", e.Container.ID, config)
		if e.LogFn != nil {
			e.LogFn("Committing changes ...")
		}
	}

	image, err := e.Client.CommitContainer(docker.CommitContainerOptions{
		Author:     b.Author,
		Container:  e.Container.ID,
		Run:        config,
		Repository: repository,
		Tag:        tag,
	})
	if err != nil {
		return fmt.Errorf("unable to commit build container: %v", err)
	}
	e.Image = image
	glog.V(4).Infof("Committed %s to %s", e.Container.ID, e.Image.ID)
	if e.LogFn != nil {
		e.LogFn("Done")
	}
	return nil
}