func (dfs *DistributedFilesystem) importImages(dirpath string, images []imagemeta, tenants map[string]struct{}) error {
for _, metadata := range images {
filename := filepath.Join(dirpath, metadata.Filename)
// Make sure all images that refer to a local registry are named with the local registry
tags := make([]string, len(metadata.Tags))
for i, tag := range metadata.Tags {
imageID, err := commons.ParseImageID(tag)
if err != nil {
glog.Errorf("Could not parse %s: %s", tag, err)
return err
}
if _, ok := tenants[imageID.User]; ok {
imageID.Host, imageID.Port = dfs.dockerHost, dfs.dockerPort
}
tags[i] = imageID.String()
}
if err := loadImage(filename, metadata.UUID, tags); err != nil {
glog.Errorf("Error loading %s (%s): %s", filename, metadata.UUID, err)
return err
}
}
return nil
}
// ImportImage creates a new image in the local repository from a file system archive.
func ImportImage(repotag, filename string) error {
dc, err := dockerclient.NewClient(dockerep)
if err != nil {
return err
}
glog.V(1).Infof("importing image %s from %s", repotag, filename)
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
iid, err := commons.ParseImageID(repotag)
if err != nil {
return err
}
opts := dockerclient.ImportImageOptions{
Repository: iid.BaseName(),
Source: "-",
InputStream: f,
Tag: iid.Tag,
}
if err = dc.ImportImage(opts); err != nil {
glog.V(1).Infof("unable to import %s: %v", repotag, err)
return err
}
return err
}
// Images returns a list of all the named images in the local repository
func Images() ([]*Image, error) {
dc, err := dockerclient.NewClient(dockerep)
if err != nil {
return nil, err
}
imgs, err := dc.ListImages(false)
if err != nil {
return nil, err
}
re := regexp.MustCompile("<none>:<none>")
resp := []*Image{}
for _, img := range imgs {
for _, repotag := range img.RepoTags {
if len(re.FindString(repotag)) > 0 {
continue
}
iid, err := commons.ParseImageID(repotag)
if err != nil {
return resp, err
}
resp = append(resp, &Image{img.ID, *iid})
}
}
return resp, nil
}
// Commit creates a new Image from the containers changes.
func (c *Container) Commit(iidstr string) (*Image, error) {
dc, err := dockerclient.NewClient(dockerep)
if err != nil {
return nil, err
}
iid, err := commons.ParseImageID(iidstr)
if err != nil {
return nil, err
}
img, err := dc.CommitContainer(
dockerclient.CommitContainerOptions{
Container: c.ID,
Repository: iid.BaseName(),
})
if err != nil {
glog.V(1).Infof("unable to commit container %s: %v", c.ID, err)
return nil, err
}
if useRegistry {
err = pushImage(iid.BaseName(), iid.Registry(), iid.Tag)
}
return &Image{img.ID, *iid}, err
}
// PushImage pushes an image by repotag to local registry, e.g., zenoss/devimg, from the local docker repository
func PushImage(repotag string) error {
iid, err := commons.ParseImageID(repotag)
if err != nil {
return err
}
return pushImage(iid.BaseName(), iid.Registry(), iid.Tag)
}
func (dfs *DistributedFilesystem) desynchronize(image *docker.Image) error {
// inspect the image
dImg, err := image.Inspect()
if err != nil {
glog.Errorf("Could not inspect image %s (%s): %s", image.ID, image.UUID, err)
return err
}
// look up services for that tenant
svcs, err := dfs.facade.GetServices(datastore.Get(), dao.ServiceRequest{TenantID: image.ID.User})
if err != nil {
glog.Errorf("Could not get services for tenant %s from %s (%s): %s", image.ID.User, image.ID, image.UUID, err)
return err
}
for _, svc := range svcs {
// figure out which services are using the provided image
svcImageID, err := commons.ParseImageID(svc.ImageID)
if err != nil {
glog.Warningf("Could not parse image %s for %s (%s): %s", svc.ImageID, svc.Name, svc.ID)
continue
} else if !svcImageID.Equals(image.ID) {
continue
}
// TODO: we need to switch to using dao.ControlPlane
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(svc.PoolID))
if err != nil {
glog.Warningf("Could not acquire connection to the coordinator (%s): %s", svc.PoolID, err)
continue
}
states, err := zkservice.GetServiceStates(conn, svc.ID)
if err != nil {
glog.Warningf("Could not get running services for %s (%s): %s", svc.Name, svc.ID)
continue
}
for _, state := range states {
// check if the instance has been running since before the commit
if state.IsRunning() && state.Started.Before(dImg.Created) {
state.InSync = false
if err := zkservice.UpdateServiceState(conn, &state); err != nil {
glog.Warningf("Could not update service state %s for %s (%s) as out of sync: %s", state.ID, svc.Name, svc.ID, err)
continue
}
}
}
}
return nil
}
// Tag tags an image in the local repository
func (img *Image) Tag(tag string) (*Image, error) {
iid, err := commons.ParseImageID(tag)
if err != nil {
return nil, err
}
dc, err := dockerclient.NewClient(dockerep)
if err != nil {
return nil, err
}
args := struct {
uuid string
name string
repo string
registry string
tag string
}{img.UUID, img.ID.String(), iid.BaseName(), iid.Registry(), iid.Tag}
glog.V(1).Infof("tagging image %s as: %s", args.repo, args.tag)
opts := dockerclient.TagImageOptions{Repo: args.repo, Tag: args.tag, Force: true}
err = dc.TagImage(args.name, opts)
if err != nil {
glog.V(1).Infof("unable to tag image %s: %v", args.repo, err)
return nil, err
}
if useRegistry {
pushImage(args.repo, args.registry, args.tag)
}
iid, err = commons.ParseImageID(fmt.Sprintf("%s:%s", args.repo, args.tag))
if err != nil {
return nil, err
}
return &Image{args.uuid, *iid}, nil
}
// ServiceUse will tag a new image (imageName) in a given registry for a given tenant
// to latest, making sure to push changes to the registry
func ServiceUse(serviceID string, imageName string, registry string, noOp bool) (string, error) {
// If noOp is True, then replace the 'real' functions that talk to Docker with
// no-op functions (for dry run purposes)
pullImage := PullImage
findImage := FindImage
tagImage := TagImage
if noOp {
pullImage = noOpPullImage
findImage = noOpFindImage
tagImage = noOpTagImage
}
// imageName is the new image to pull, eg. "zenoss/resmgr-unstable:1.2.3.4"
glog.V(0).Infof("preparing to use image: %s", imageName)
imageID, err := commons.ParseImageID(imageName)
if err != nil {
return "", err
}
if imageID.Tag == "" {
imageID.Tag = "latest"
}
glog.Infof("pulling image %s, this may take a while...", imageID)
if err := pullImage(imageID.String()); err != nil {
glog.Warningf("unable to pull image %s", imageID)
}
//verify image has been pulled
img, err := findImage(imageID.String(), false)
if err != nil {
err = fmt.Errorf("could not look up image %s: %s. Check your docker login and retry service deployment.", imageID, err)
return "", err
}
//Tag images to latest all images
var newTag *commons.ImageID
newTag, err = commons.RenameImageID(registry, serviceID, imageID.String(), "latest")
if err != nil {
return "", err
}
glog.Infof("tagging image %s to %s ", imageName, newTag)
if _, err = tagImage(img, newTag.String()); err != nil {
glog.Errorf("could not tag image: %s (%v)", imageName, err)
return "", err
}
return newTag.String(), nil
}
func pullTemplateImages(template *servicetemplate.ServiceTemplate) error {
for _, img := range getImageIDs(template.Services...) {
imageID, err := commons.ParseImageID(img)
if err != nil {
return err
}
tag := imageID.Tag
if tag == "" {
tag = "latest"
}
image := fmt.Sprintf("%s:%s", imageID.BaseName(), tag)
glog.Infof("Pulling image %s", image)
if err := docker.PullImage(image); err != nil {
glog.Warningf("Unable to pull image %s", image)
}
}
return nil
}
// ResetRegistry will update the host:port of the docker registry
func (dfs *DistributedFilesystem) ResetRegistry() error {
// get all the services in the system
svcs, err := dfs.facade.GetServices(datastore.Get(), dao.ServiceRequest{})
if err != nil {
glog.Errorf("Could not get services for updating the registry")
return err
}
imagemap := make(map[string]struct{})
for _, svc := range svcs {
imageID, err := commons.ParseImageID(svc.ImageID)
if err != nil {
glog.Errorf("Could not parse image ID (%s) for service %s (%s): %s", svc.ImageID, svc.Name, svc.ID, err)
return err
}
if imageID.Host == dfs.dockerHost && imageID.Port == dfs.dockerPort {
continue
}
if _, ok := imagemap[imageID.BaseName()]; !ok {
if err := dfs.registerImages(imageID.BaseName()); err != nil {
glog.Errorf("Could not reregister image %s: %s", imageID.BaseName(), err)
return err
}
imagemap[imageID.BaseName()] = struct{}{}
}
imageID.Host, imageID.Port = dfs.dockerHost, dfs.dockerPort
svc.ImageID = imageID.String()
if err := dfs.facade.UpdateService(datastore.Get(), svc); err != nil {
glog.Errorf("Could not update service %s (%s) with image %s", svc.Name, svc.ID, svc.ImageID)
return err
}
}
return nil
}
func lookupImage(repotag string) (*Image, error) {
imgs, err := Images()
if err != nil {
return nil, err
}
iid, err := commons.ParseImageID(repotag)
if err != nil {
return nil, err
}
if len(iid.Tag) == 0 {
repotag = repotag + ":latest"
}
for _, img := range imgs {
if img.ID.String() == repotag {
glog.V(1).Info("found: ", repotag)
return img, nil
}
}
return nil, ErrNoSuchImage
}
// configureContainer creates and populates two structures, a docker client Config and a docker client HostConfig structure
// that are used to create and start a container respectively. The information used to populate the structures is pulled from
// the service, serviceState, and conn values that are passed into configureContainer.
func configureContainer(a *HostAgent, client *ControlClient,
svc *service.Service, serviceState *servicestate.ServiceState,
virtualAddressSubnet string) (*dockerclient.Config, *dockerclient.HostConfig, error) {
cfg := &dockerclient.Config{}
hcfg := &dockerclient.HostConfig{}
//get this service's tenantId for volume mapping
var tenantID string
err := client.GetTenantId(svc.ID, &tenantID)
if err != nil {
glog.Errorf("Failed getting tenantID for service: %s, %s", svc.ID, err)
return nil, nil, err
}
// get the system user
unused := 0
systemUser := user.User{}
err = client.GetSystemUser(unused, &systemUser)
if err != nil {
glog.Errorf("Unable to get system user account for agent %s", err)
return nil, nil, err
}
glog.V(1).Infof("System User %v", systemUser)
cfg.Image = svc.ImageID
// get the endpoints
cfg.ExposedPorts = make(map[dockerclient.Port]struct{})
hcfg.PortBindings = make(map[dockerclient.Port][]dockerclient.PortBinding)
if svc.Endpoints != nil {
glog.V(1).Info("Endpoints for service: ", svc.Endpoints)
for _, endpoint := range svc.Endpoints {
if endpoint.Purpose == "export" { // only expose remote endpoints
var port uint16
port = endpoint.PortNumber
if endpoint.PortTemplate != "" {
t := template.Must(template.New("PortTemplate").Funcs(funcmap).Parse(endpoint.PortTemplate))
b := bytes.Buffer{}
err := t.Execute(&b, serviceState)
if err == nil {
j, err := strconv.Atoi(b.String())
if err != nil {
glog.Errorf("%+v", err)
} else if j > 0 {
port = uint16(j)
}
}
}
var p string
switch endpoint.Protocol {
case commons.UDP:
p = fmt.Sprintf("%d/%s", port, "udp")
default:
p = fmt.Sprintf("%d/%s", port, "tcp")
}
cfg.ExposedPorts[dockerclient.Port(p)] = struct{}{}
hcfg.PortBindings[dockerclient.Port(p)] = append(hcfg.PortBindings[dockerclient.Port(p)], dockerclient.PortBinding{})
}
}
}
if len(tenantID) == 0 && len(svc.Volumes) > 0 {
// FIXME: find a better way of handling this error condition
glog.Fatalf("Could not get tenant ID and need to mount a volume, service state: %s, service id: %s", serviceState.ID, svc.ID)
}
// Make sure the image exists locally.
if _, err = docker.FindImage(svc.ImageID, true); err != nil {
glog.Errorf("can't find docker image %s: %s", svc.ImageID, err)
return nil, nil, err
}
cfg.Volumes = make(map[string]struct{})
hcfg.Binds = []string{}
if err := injectContext(svc, serviceState, client); err != nil {
glog.Errorf("Error injecting context: %s", err)
return nil, nil, err
}
for _, volume := range svc.Volumes {
if volume.Type != "" && volume.Type != "dfs" {
continue
}
resourcePath, err := a.setupVolume(tenantID, svc, volume)
if err != nil {
return nil, nil, err
}
binding := fmt.Sprintf("%s:%s", resourcePath, volume.ContainerPath)
cfg.Volumes[strings.Split(binding, ":")[1]] = struct{}{}
hcfg.Binds = append(hcfg.Binds, strings.TrimSpace(binding))
}
dir, binary, err := ExecPath()
if err != nil {
glog.Errorf("Error getting exec path: %v", err)
return nil, nil, err
}
volumeBinding := fmt.Sprintf("%s:/serviced", dir)
cfg.Volumes[strings.Split(volumeBinding, ":")[1]] = struct{}{}
hcfg.Binds = append(hcfg.Binds, strings.TrimSpace(volumeBinding))
// bind mount everything we need for logstash-forwarder
if len(svc.LogConfigs) != 0 {
const LOGSTASH_CONTAINER_DIRECTORY = "/usr/local/serviced/resources/logstash"
logstashPath := utils.ResourcesDir() + "/logstash"
binding := fmt.Sprintf("%s:%s", logstashPath, LOGSTASH_CONTAINER_DIRECTORY)
cfg.Volumes[LOGSTASH_CONTAINER_DIRECTORY] = struct{}{}
hcfg.Binds = append(hcfg.Binds, binding)
glog.V(1).Infof("added logstash bind mount: %s", binding)
}
// specify temporary volume paths for docker to create
tmpVolumes := []string{"/tmp"}
for _, volume := range svc.Volumes {
if volume.Type == "tmp" {
tmpVolumes = append(tmpVolumes, volume.ContainerPath)
}
}
for _, path := range tmpVolumes {
cfg.Volumes[path] = struct{}{}
glog.V(4).Infof("added temporary docker container path: %s", path)
}
// add arguments to mount requested directory (if requested)
glog.V(2).Infof("Checking Mount options for service %#v", svc)
for _, bindMountString := range a.mount {
glog.V(2).Infof("bindmount is %#v", bindMountString)
splitMount := strings.Split(bindMountString, ",")
numMountArgs := len(splitMount)
if numMountArgs == 2 || numMountArgs == 3 {
requestedImage := splitMount[0]
glog.V(2).Infof("mount requestedImage %#v", requestedImage)
hostPath := splitMount[1]
glog.V(2).Infof("mount hostPath %#v", hostPath)
// assume the container path is going to be the same as the host path
containerPath := hostPath
// if the container path is provided, use it
if numMountArgs > 2 {
containerPath = splitMount[2]
}
glog.V(2).Infof("mount containerPath %#v", containerPath)
// insert tenantId into requestedImage - see facade.DeployService
matchedRequestedImage := false
if requestedImage == "*" {
matchedRequestedImage = true
} else {
imageID, err := commons.ParseImageID(requestedImage)
if err != nil {
glog.Errorf("error parsing imageid %v: %v", requestedImage, err)
continue
}
svcImageID, err := commons.ParseImageID(svc.ImageID)
if err != nil {
glog.Errorf("error parsing service imageid %v; %v", svc.ImageID, err)
continue
}
glog.V(2).Infof("mount checking %#v and %#v ", imageID, svcImageID)
matchedRequestedImage = (imageID.Repo == svcImageID.Repo)
}
if matchedRequestedImage {
binding := fmt.Sprintf("%s:%s", hostPath, containerPath)
cfg.Volumes[strings.Split(binding, ":")[1]] = struct{}{}
hcfg.Binds = append(hcfg.Binds, strings.TrimSpace(binding))
}
} else {
glog.Warningf("Could not bind mount the following: %s", bindMountString)
}
}
// Get host IP
ips, err := utils.GetIPv4Addresses()
if err != nil {
glog.Errorf("Error getting host IP addresses: %v", err)
return nil, nil, err
}
// add arguments for environment variables
cfg.Env = append([]string{},
fmt.Sprintf("CONTROLPLANE_SYSTEM_USER=%s", systemUser.Name),
fmt.Sprintf("CONTROLPLANE_SYSTEM_PASSWORD=%s", systemUser.Password),
fmt.Sprintf("CONTROLPLANE_HOST_IPS='%s'", strings.Join(ips, " ")),
fmt.Sprintf("SERVICED_VIRTUAL_ADDRESS_SUBNET=%s", virtualAddressSubnet),
fmt.Sprintf("SERVICED_IS_SERVICE_SHELL=false"),
fmt.Sprintf("SERVICED_NOREGISTRY=%s", os.Getenv("SERVICED_NOREGISTRY")),
fmt.Sprintf("SERVICED_SERVICE_IMAGE=%s", svc.ImageID),
fmt.Sprintf("SERVICED_MAX_RPC_CLIENTS=1"),
fmt.Sprintf("SERVICED_RPC_PORT=%s", a.rpcport),
fmt.Sprintf("TZ=%s", os.Getenv("TZ")))
// add dns values to setup
for _, addr := range a.dockerDNS {
_addr := strings.TrimSpace(addr)
if len(_addr) > 0 {
cfg.Dns = append(cfg.Dns, addr)
}
}
// Add hostname if set
if svc.Hostname != "" {
cfg.Hostname = svc.Hostname
}
cfg.Cmd = append([]string{},
fmt.Sprintf("/serviced/%s", binary),
"service",
"proxy",
svc.ID,
strconv.Itoa(serviceState.InstanceID),
svc.Startup)
if svc.Privileged {
hcfg.Privileged = true
}
// Memory and CpuShares should never be negative
if svc.MemoryLimit < 0 {
cfg.Memory = 0
} else {
cfg.Memory = svc.MemoryLimit
}
if svc.CPUShares < 0 {
cfg.CpuShares = 0
} else {
cfg.CpuShares = svc.CPUShares
}
return cfg, hcfg, nil
}
// NewContainer creates a new container and returns its id. The supplied create action, if
// any, will be executed on successful creation of the container. If a start action is specified
// it will be executed after the container has been started. Note, if the start parameter is
// false the container won't be started and the start action will not be executed.
func NewContainer(cd *ContainerDefinition, start bool, timeout time.Duration, oncreate ContainerActionFunc, onstart ContainerActionFunc) (*Container, error) {
args := struct {
containerOptions *dockerclient.CreateContainerOptions
hostConfig *dockerclient.HostConfig
start bool
createaction ContainerActionFunc
startaction ContainerActionFunc
}{&cd.CreateContainerOptions, &cd.HostConfig, start, oncreate, onstart}
timeoutc := time.After(timeout)
dc, err := dockerclient.NewClient(dockerep)
if err != nil {
return nil, err
}
em, err := dc.MonitorEvents()
if err != nil {
return nil, fmt.Errorf("can't monitor Docker events: %v", err)
}
iid, err := commons.ParseImageID(args.containerOptions.Config.Image)
if err != nil {
return nil, err
}
if useRegistry {
if err := PullImage(iid.String()); err != nil {
glog.V(2).Infof("Unable to pull image %s: %v", iid.String(), err)
if _, err2 := lookupImage(iid.String()); err2 != nil {
return nil, err2
}
}
}
glog.V(2).Infof("creating container: %#v", *args.containerOptions)
ctr, err := dc.CreateContainer(*args.containerOptions)
switch {
case err == dockerclient.ErrNoSuchImage:
if err := PullImage(iid.String()); err != nil {
glog.V(2).Infof("Unable to pull image %s: %v", iid.String(), err)
return nil, err
}
ctr, err = dc.CreateContainer(*args.containerOptions)
if err != nil {
glog.V(2).Infof("container creation failed %+v: %v", *args.containerOptions, err)
return nil, err
}
case err != nil:
glog.V(2).Infof("container creation failed %+v: %v", *args.containerOptions, err)
return nil, err
}
glog.V(2).Infof("created container: %+v", *ctr)
if args.createaction != nil {
args.createaction(ctr.ID)
}
if args.start {
ss, err := em.Subscribe(ctr.ID)
if err != nil {
return nil, err
}
sc := make(chan struct{})
ss.Handle(Start, func(e dockerclient.Event) error {
if args.startaction != nil {
args.startaction(ctr.ID)
}
glog.V(2).Infof("handling event: %+v for %s", e, ctr.ID)
close(sc)
return nil
})
defer ss.Cancel()
glog.V(2).Infof("post creation start of %s: %+v", ctr.ID, args.hostConfig)
err = dc.StartContainer(ctr.ID, args.hostConfig)
if err != nil {
glog.V(1).Infof("post creation start of %s failed: %v", ctr.ID, err)
return nil, err
}
glog.V(2).Infof("======= wait for %s to start =======", ctr.ID)
attempts := 0
WaitForContainerStart:
for {
select {
case <-timeoutc:
glog.V(2).Infof("timeout starting container")
return nil, fmt.Errorf("docker timeout starting container after %s", timeout)
case <-sc:
glog.V(2).Infof("update container %s state post start", ctr.ID)
ctrID := ctr.ID
ctr, err = dc.InspectContainer(ctrID)
if err != nil {
glog.V(1).Infof("failed to update container %s state post start: %v", ctrID, err)
return nil, err
}
glog.V(2).Infof("container %s is started", ctr.ID)
break WaitForContainerStart
case <-time.After(5 * time.Second):
nctr, err := dc.InspectContainer(ctr.ID)
if err != nil {
glog.V(2).Infof("can't inspect container %s: %v", ctr.ID, err)
return nil, err
}
ctr = nctr
switch {
case !ctr.State.Running && attempts > maxStartAttempts:
glog.V(2).Infof("timed out starting container")
return nil, fmt.Errorf("timed out starting container: %s", ctr.ID)
case !ctr.State.Running:
attempts = attempts + 1
continue WaitForContainerStart
default:
glog.V(2).Infof("container %s is running", ctr.ID)
break WaitForContainerStart
}
}
}
}
return &Container{ctr, cd.HostConfig}, nil
}
