func (l *LocalCluster) runDockerSpy() {
l.panicOnStop()
create := func() (*Container, error) {
return createContainer(l,
container.Config{
Image: dockerspyImage + ":" + dockerspyTag,
Cmd: strslice.New("--dns-domain=" + domain),
}, container.HostConfig{
Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
PublishAllPorts: true,
},
"docker-spy",
)
}
c, err := create()
if dockerclient.IsErrImageNotFound(err) {
if err := pullImage(l, types.ImagePullOptions{ImageID: dockerspyImage, Tag: dockerspyTag}); err != nil {
log.Fatal(err)
}
c, err = create()
}
maybePanic(err)
maybePanic(c.Start())
l.dns = c
if ci, err := c.Inspect(); err != nil {
log.Error(err)
} else {
log.Infof("started %s: %s", c.Name(), ci.NetworkSettings.IPAddress)
}
}
// CreateContainer aka schedule a brand new container into the cluster.
func (c *Cluster) CreateContainer(config *cluster.ContainerConfig, name string, authConfig *types.AuthConfig) (*cluster.Container, error) {
poolUUID := os.Getenv("CX_POOL")
log.Debug("CX: Adding label pool=" + poolUUID)
config.Labels["pool"] = poolUUID
container, err := c.createContainer(config, name, false, authConfig)
if err != nil {
var retries int64
// fails with image not found, then try to reschedule with image affinity
// ENGINEAPIFIXME: The first error can be removed once dockerclient is removed
bImageNotFoundError, _ := regexp.MatchString(`image \S* not found`, err.Error())
if (bImageNotFoundError || client.IsErrImageNotFound(err)) && !config.HaveNodeConstraint() {
// Check if the image exists in the cluster
// If exists, retry with an image affinity
if c.Image(config.Image) != nil {
container, err = c.createContainer(config, name, true, authConfig)
retries++
}
}
for ; retries < c.createRetry && err != nil; retries++ {
log.WithFields(log.Fields{"Name": "Swarm"}).Warnf("Failed to create container: %s, retrying", err)
container, err = c.createContainer(config, name, false, authConfig)
}
}
return container, err
}
// OutOfSync checks if the container is out of sync with the service definition.
// It looks if the the service hash container label is the same as the computed one.
func (c *Container) OutOfSync(imageName string) (bool, error) {
info, err := c.findInfo()
if err != nil {
return false, err
}
if info.Config.Image != imageName {
logrus.Debugf("Images for %s do not match %s!=%s", c.name, info.Config.Image, imageName)
return true, nil
}
if info.Config.Labels[HASH.Str()] != c.getHash() {
logrus.Debugf("Hashes for %s do not match %s!=%s", c.name, info.Config.Labels[HASH.Str()], c.getHash())
return true, nil
}
image, _, err := c.client.ImageInspectWithRaw(context.Background(), info.Config.Image, false)
if err != nil {
if client.IsErrImageNotFound(err) {
logrus.Debugf("Image %s do not exist, do not know if it's out of sync", info.Config.Image)
return false, nil
}
return false, err
}
logrus.Debugf("Checking existing image name vs id: %s == %s", image.ID, info.Image)
return image.ID != info.Image, err
}
func (cli *DockerCli) inspectAll(ctx context.Context, getSize bool) inspect.GetRefFunc {
return func(ref string) (interface{}, []byte, error) {
c, rawContainer, err := cli.client.ContainerInspectWithRaw(ctx, ref, getSize)
if err != nil {
// Search for image with that id if a container doesn't exist.
if client.IsErrContainerNotFound(err) {
i, rawImage, err := cli.client.ImageInspectWithRaw(ctx, ref, getSize)
if err != nil {
if client.IsErrImageNotFound(err) {
// Search for task with that id if an image doesn't exists.
t, rawTask, err := cli.client.TaskInspectWithRaw(ctx, ref)
if err != nil {
return nil, nil, fmt.Errorf("Error: No such image, container or task: %s", ref)
}
if getSize {
fmt.Fprintln(cli.err, "WARNING: --size ignored for tasks")
}
return t, rawTask, nil
}
return nil, nil, err
}
return i, rawImage, err
}
return nil, nil, err
}
return c, rawContainer, err
}
}
func (c *Container) createContainer(imageName, oldContainer string, configOverride *config.ServiceConfig) (*types.Container, error) {
serviceConfig := c.service.serviceConfig
if configOverride != nil {
serviceConfig.Command = configOverride.Command
serviceConfig.Tty = configOverride.Tty
serviceConfig.StdinOpen = configOverride.StdinOpen
}
configWrapper, err := ConvertToAPI(c.service)
if err != nil {
return nil, err
}
configWrapper.Config.Image = imageName
if configWrapper.Config.Labels == nil {
configWrapper.Config.Labels = map[string]string{}
}
configWrapper.Config.Labels[SERVICE.Str()] = c.service.name
configWrapper.Config.Labels[PROJECT.Str()] = c.service.context.Project.Name
configWrapper.Config.Labels[HASH.Str()] = c.getHash()
// libcompose run command not yet supported, so always "False"
configWrapper.Config.Labels[ONEOFF.Str()] = "False"
configWrapper.Config.Labels[NUMBER.Str()] = getContainerNumber(c)
configWrapper.Config.Labels[VERSION.Str()] = ComposeVersion
err = c.populateAdditionalHostConfig(configWrapper.HostConfig)
if err != nil {
return nil, err
}
if oldContainer != "" {
info, err := c.client.ContainerInspect(context.Background(), oldContainer)
if err != nil {
return nil, err
}
configWrapper.HostConfig.Binds = util.Merge(configWrapper.HostConfig.Binds, volumeBinds(configWrapper.Config.Volumes, &info))
}
logrus.Debugf("Creating container %s %#v", c.name, configWrapper)
container, err := c.client.ContainerCreate(context.Background(), configWrapper.Config, configWrapper.HostConfig, configWrapper.NetworkingConfig, c.name)
if err != nil {
if client.IsErrImageNotFound(err) {
logrus.Debugf("Not Found, pulling image %s", configWrapper.Config.Image)
if err = c.pull(configWrapper.Config.Image); err != nil {
return nil, err
}
if container, err = c.client.ContainerCreate(context.Background(), configWrapper.Config, configWrapper.HostConfig, configWrapper.NetworkingConfig, c.name); err != nil {
return nil, err
}
} else {
logrus.Debugf("Failed to create container %s: %v", c.name, err)
return nil, err
}
}
return GetContainerByID(c.client, container.ID)
}
func ensureImage(cli DockerClient, image string) (string, error) {
ctx := context.Background()
info, _, err := cli.ImageInspectWithRaw(ctx, image, false)
if err == nil {
logrus.Debugf("Image found locally %s", image)
return info.ID, nil
}
if !client.IsErrImageNotFound(err) {
logrus.Errorf("Error inspecting image %q: %v", image, err)
return "", err
}
// Image must be tagged reference if it does not exist
ref, err := reference.Parse(image)
if err != nil {
logrus.Errorf("Image is not valid reference %q: %v", image, err)
return "", err
}
tagged, ok := ref.(reference.NamedTagged)
if !ok {
logrus.Errorf("Tagged reference required %q", image)
return "", errors.New("invalid reference, tag needed")
}
pullStart := time.Now()
pullOptions := types.ImagePullOptions{
PrivilegeFunc: registryAuthNotSupported,
}
resp, err := cli.ImagePull(ctx, tagged.String(), pullOptions)
if err != nil {
logrus.Errorf("Error pulling image %q: %v", tagged.String(), err)
return "", err
}
defer resp.Close()
outFd, isTerminalOut := term.GetFdInfo(os.Stdout)
if err = jsonmessage.DisplayJSONMessagesStream(resp, os.Stdout, outFd, isTerminalOut, nil); err != nil {
logrus.Errorf("Error copying pull output: %v", err)
return "", err
}
// TODO: Get pulled digest
logFields := logrus.Fields{
timerKey: time.Since(pullStart),
"image": tagged.String(),
}
logrus.WithFields(logFields).Info("image pulled")
info, _, err = cli.ImageInspectWithRaw(ctx, tagged.String(), false)
if err != nil {
return "", nil
}
return info.ID, nil
}
func (d *kubeDockerClient) InspectImage(image string) (*dockertypes.ImageInspect, error) {
resp, _, err := d.client.ImageInspectWithRaw(getDefaultContext(), image, true)
if err != nil {
if dockerapi.IsErrImageNotFound(err) {
err = imageNotFoundError{ID: image}
}
return nil, err
}
return &resp, nil
}
// IsImageInLocalRegistry determines whether the supplied image is in the local registry.
func (d *stiDocker) IsImageInLocalRegistry(name string) (bool, error) {
name = getImageName(name)
resp, err := d.InspectImage(name)
if resp != nil {
return true, nil
}
if err != nil && !dockerapi.IsErrImageNotFound(err) {
return false, errors.NewInspectImageError(name, err)
}
return false, nil
}
func (s stiDocker) InspectImage(name string) (*dockertypes.ImageInspect, error) {
ctx, _ := context.WithTimeout(context.Background(), 2*time.Minute)
resp, _, err := s.client.ImageInspectWithRaw(ctx, name, true)
if err != nil {
if dockerapi.IsErrImageNotFound(err) {
return nil, fmt.Errorf("no such image :%q", name)
}
return nil, err
}
return &resp, nil
}
func (i integrationTest) InspectImage(name string) (*dockertypes.ImageInspect, error) {
ctx, cancel := getDefaultContext()
defer cancel()
resp, _, err := i.engineClient.ImageInspectWithRaw(ctx, name)
if err != nil {
if dockerapi.IsErrImageNotFound(err) {
return nil, fmt.Errorf("no such image :%q", name)
}
return nil, err
}
return &resp, nil
}
func (cli *DockerCli) createContainer(config *container.Config, hostConfig *container.HostConfig, networkingConfig *networktypes.NetworkingConfig, cidfile, name string) (*types.ContainerCreateResponse, error) {
var containerIDFile *cidFile
if cidfile != "" {
var err error
if containerIDFile, err = newCIDFile(cidfile); err != nil {
return nil, err
}
defer containerIDFile.Close()
}
_, ref, err := reference.ParseIDOrReference(config.Image)
if err != nil {
return nil, err
}
if ref != nil {
ref = reference.WithDefaultTag(ref)
}
//create the container
response, err := cli.client.ContainerCreate(context.Background(), config, hostConfig, networkingConfig, name)
//if image not found try to pull it
if err != nil {
if client.IsErrImageNotFound(err) && ref != nil {
fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", ref.String())
// we don't want to write to stdout anything apart from container.ID
if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil {
return nil, err
}
// Retry
var retryErr error
response, retryErr = cli.client.ContainerCreate(context.Background(), config, hostConfig, networkingConfig, name)
if retryErr != nil {
return nil, retryErr
}
} else {
return nil, err
}
}
for _, warning := range response.Warnings {
fmt.Fprintf(cli.err, "WARNING: %s\n", warning)
}
if containerIDFile != nil {
if err = containerIDFile.Write(response.ID); err != nil {
return nil, err
}
}
return &response, nil
}
func (c *Container) createContainer(imageName, oldContainer string) (*types.Container, error) {
configWrapper, err := ConvertToAPI(c.service)
if err != nil {
return nil, err
}
configWrapper.Config.Image = imageName
if configWrapper.Config.Labels == nil {
configWrapper.Config.Labels = map[string]string{}
}
configWrapper.Config.Labels[NAME.Str()] = c.name
configWrapper.Config.Labels[SERVICE.Str()] = c.service.name
configWrapper.Config.Labels[PROJECT.Str()] = c.service.context.Project.Name
configWrapper.Config.Labels[HASH.Str()] = c.getHash()
err = c.populateAdditionalHostConfig(configWrapper.HostConfig)
if err != nil {
return nil, err
}
if oldContainer != "" {
info, err := c.client.ContainerInspect(context.Background(), oldContainer)
if err != nil {
return nil, err
}
configWrapper.HostConfig.Binds = util.Merge(configWrapper.HostConfig.Binds, volumeBinds(configWrapper.Config.Volumes, &info))
}
logrus.Debugf("Creating container %s %#v", c.name, configWrapper)
container, err := c.client.ContainerCreate(context.Background(), configWrapper.Config, configWrapper.HostConfig, configWrapper.NetworkingConfig, c.name)
if err != nil {
if client.IsErrImageNotFound(err) {
logrus.Debugf("Not Found, pulling image %s", configWrapper.Config.Image)
if err = c.pull(configWrapper.Config.Image); err != nil {
return nil, err
}
if container, err = c.client.ContainerCreate(context.Background(), configWrapper.Config, configWrapper.HostConfig, configWrapper.NetworkingConfig, c.name); err != nil {
return nil, err
}
} else {
logrus.Debugf("Failed to create container %s: %v", c.name, err)
return nil, err
}
}
return GetContainerByID(c.client, container.ID)
}
// Create a new container
func (e *Engine) Create(config *ContainerConfig, name string, pullImage bool, authConfig *types.AuthConfig) (*Container, error) {
var (
err error
createResp types.ContainerCreateResponse
)
// Convert our internal ContainerConfig into something Docker will
// understand. Start by making a copy of the internal ContainerConfig as
// we don't want to mess with the original.
dockerConfig := *config
// nb of CPUs -> real CpuShares
// FIXME remove "duplicate" lines and move this to cluster/config.go
dockerConfig.HostConfig.CPUShares = int64(math.Ceil(float64(config.HostConfig.CPUShares*1024) / float64(e.Cpus)))
createResp, err = e.apiClient.ContainerCreate(context.Background(), &dockerConfig.Config, &dockerConfig.HostConfig, &dockerConfig.NetworkingConfig, name)
e.CheckConnectionErr(err)
if err != nil {
// If the error is other than not found, abort immediately.
if (err != dockerclient.ErrImageNotFound && !engineapi.IsErrImageNotFound(err)) || !pullImage {
return nil, err
}
// Otherwise, try to pull the image...
if err = e.Pull(config.Image, authConfig); err != nil {
return nil, err
}
// ...And try again.
createResp, err = e.apiClient.ContainerCreate(context.Background(), &dockerConfig.Config, &dockerConfig.HostConfig, &dockerConfig.NetworkingConfig, name)
e.CheckConnectionErr(err)
if err != nil {
return nil, err
}
}
// Register the container immediately while waiting for a state refresh.
// Force a state refresh to pick up the newly created container.
e.refreshContainer(createResp.ID, true)
e.RefreshVolumes()
e.RefreshNetworks()
e.Lock()
container := e.containers[createResp.ID]
e.Unlock()
if container == nil {
err = errors.New("Container created but refresh didn't report it back")
}
return container, err
}
func (d *kubeDockerClient) InspectImage(image string) (*dockertypes.ImageInspect, error) {
ctx, cancel := d.getTimeoutContext()
defer cancel()
resp, _, err := d.client.ImageInspectWithRaw(ctx, image, true)
if ctxErr := contextError(ctx); ctxErr != nil {
return nil, ctxErr
}
if err != nil {
if dockerapi.IsErrImageNotFound(err) {
err = imageNotFoundError{ID: image}
}
return nil, err
}
return &resp, nil
}
func (d *kubeDockerClient) InspectImage(image string) (*docker.Image, error) {
resp, _, err := d.client.ImageInspectWithRaw(getDefaultContext(), image, true)
if err != nil {
// TODO(random-liu): Use IsErrImageNotFound instead of ErrNoSuchImage
if dockerapi.IsErrImageNotFound(err) {
err = docker.ErrNoSuchImage
}
return nil, err
}
imageInfo := &docker.Image{}
if err := convertType(&resp, imageInfo); err != nil {
return nil, err
}
return imageInfo, nil
}
func (s *Service) ensureImageExists() (string, error) {
err := s.imageExists()
if err == nil {
return s.imageName(), nil
}
if err != nil && !client.IsErrImageNotFound(err) {
return "", err
}
if s.Config().Build != "" {
if s.context.NoBuild {
return "", fmt.Errorf("Service %q needs to be built, but no-build was specified", s.name)
}
return s.imageName(), s.build()
}
return s.imageName(), s.Pull()
}
func (cli *DockerCli) inspectAll(getSize bool) inspectSearcher {
return func(ref string) (interface{}, []byte, error) {
c, rawContainer, err := cli.client.ContainerInspectWithRaw(ref, getSize)
if err != nil {
// Search for image with that id if a container doesn't exist.
if client.IsErrContainerNotFound(err) {
i, rawImage, err := cli.client.ImageInspectWithRaw(ref, getSize)
if err != nil {
if client.IsErrImageNotFound(err) {
return nil, nil, fmt.Errorf("Error: No such image or container: %s", ref)
}
return nil, nil, err
}
return i, rawImage, err
}
return nil, nil, err
}
return c, rawContainer, err
}
}
func (p *Project) ensureImageExists(ref string, force bool) error {
if !force {
// Check if ref is already there
_, _, err := p.Client.ImageInspectWithRaw(context.Background(), ref, false)
if err != nil && !client.IsErrImageNotFound(err) {
return err
}
if err == nil {
return nil
}
}
// And pull it
responseBody, err := p.Client.ImagePull(context.Background(), ref, types.ImagePullOptions{})
if err != nil {
fmt.Printf("%v", err)
return err
}
defer responseBody.Close()
_, err = ioutil.ReadAll(responseBody)
return err
}
func createContainer(ctx context.Context, dockerCli *client.DockerCli, config *container.Config, hostConfig *container.HostConfig, networkingConfig *networktypes.NetworkingConfig, cidfile, name string) (*types.ContainerCreateResponse, error) {
stderr := dockerCli.Err()
var containerIDFile *cidFile
if cidfile != "" {
var err error
if containerIDFile, err = newCIDFile(cidfile); err != nil {
return nil, err
}
defer containerIDFile.Close()
}
var trustedRef reference.Canonical
_, ref, err := reference.ParseIDOrReference(config.Image)
if err != nil {
return nil, err
}
if ref != nil {
ref = reference.WithDefaultTag(ref)
if ref, ok := ref.(reference.NamedTagged); ok && client.IsTrusted() {
var err error
trustedRef, err = dockerCli.TrustedReference(ctx, ref)
if err != nil {
return nil, err
}
config.Image = trustedRef.String()
}
}
//create the container
response, err := dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, name)
//if image not found try to pull it
if err != nil {
if apiclient.IsErrImageNotFound(err) && ref != nil {
fmt.Fprintf(stderr, "Unable to find image '%s' locally\n", ref.String())
// we don't want to write to stdout anything apart from container.ID
if err = pullImage(ctx, dockerCli, config.Image, stderr); err != nil {
return nil, err
}
if ref, ok := ref.(reference.NamedTagged); ok && trustedRef != nil {
if err := dockerCli.TagTrusted(ctx, trustedRef, ref); err != nil {
return nil, err
}
}
// Retry
var retryErr error
response, retryErr = dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, name)
if retryErr != nil {
return nil, retryErr
}
} else {
return nil, err
}
}
for _, warning := range response.Warnings {
fmt.Fprintf(stderr, "WARNING: %s\n", warning)
}
if containerIDFile != nil {
if err = containerIDFile.Write(response.ID); err != nil {
return nil, err
}
}
return &response, nil
}
// create the volumes container that keeps all of the volumes used by
// the cluster.
func (l *LocalCluster) initCluster() {
log.Infof("initializing cluster")
l.panicOnStop()
// Create the temporary certs directory in the current working
// directory. Boot2docker's handling of binding local directories
// into the container is very confusing. If the directory being
// bound has a parent directory that exists in the boot2docker VM
// then that directory is bound into the container. In particular,
// that means that binds of /tmp and /var will be problematic.
var err error
l.CertsDir, err = ioutil.TempDir(pwd, ".localcluster.certs.")
maybePanic(err)
binds := []string{
l.CertsDir + ":/certs",
filepath.Join(pwd, "..") + ":/go/src/github.com/cockroachdb/cockroach",
}
if l.logDir != "" {
l.keepLogs = true
if !filepath.IsAbs(l.logDir) {
l.logDir = filepath.Join(pwd, l.logDir)
}
binds = append(binds, l.logDir+":/logs")
// If we don't make sure the directory exists, Docker will and then we
// may run into ownership issues (think Docker running as root, but us
// running as a regular Joe as it happens on CircleCI).
maybePanic(os.MkdirAll(l.logDir, 0777))
}
if *cockroachImage == builderImage {
path, err := filepath.Abs(*cockroachBinary)
maybePanic(err)
binds = append(binds, path+":/"+filepath.Base(*cockroachBinary))
}
vols := map[string]struct{}{}
for i := 0; i < l.numLocal; i++ {
for j := 0; j < l.numStores; j++ {
vols[dataStr(i, j)] = struct{}{}
}
}
create := func() (*Container, error) {
return createContainer(
l,
container.Config{
Image: *cockroachImage,
Volumes: vols,
Entrypoint: strslice.New("/bin/true"),
}, container.HostConfig{
Binds: binds,
PublishAllPorts: true,
},
"volumes",
)
}
c, err := create()
if dockerclient.IsErrImageNotFound(err) && *cockroachImage == builderImage {
if err := pullImage(l, types.ImagePullOptions{ImageID: *cockroachImage}); err != nil {
log.Fatal(err)
}
c, err = create()
}
maybePanic(err)
maybePanic(c.Start())
maybePanic(c.Wait())
l.vols = c
}