// createImage creates a docker image either by pulling it from a registry or by
// loading it from the file system
func (d *DockerDriver) createImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir string) error {
image := driverConfig.ImageName
repo, tag := docker.ParseRepositoryTag(image)
if tag == "" {
tag = "latest"
}
var dockerImage *docker.Image
var err error
// We're going to check whether the image is already downloaded. If the tag
// is "latest" we have to check for a new version every time so we don't
// bother to check and cache the id here. We'll download first, then cache.
if tag != "latest" {
dockerImage, err = client.InspectImage(image)
}
// Download the image
if dockerImage == nil {
if len(driverConfig.LoadImages) > 0 {
return d.loadImage(driverConfig, client, taskDir)
}
return d.pullImage(driverConfig, client, repo, tag)
}
return err
}
func (cli *DogestryCli) getLayerIdsToDownload(fromId remote.ID, imageRoot string, r remote.Remote, client *docker.Client) ([]remote.ID, error) {
toDownload := make([]remote.ID, 0)
err := r.WalkImages(fromId, func(id remote.ID, image docker.Image, err error) error {
fmt.Printf("Examining id '%s' on remote docker host...\n", id.Short())
if err != nil {
return err
}
_, err = client.InspectImage(string(id))
if err == docker.ErrNoSuchImage {
toDownload = append(toDownload, id)
return nil
} else if err != nil {
return err
} else {
fmt.Printf("Docker host already has id '%s', stop scanning.\n", id.Short())
return remote.BreakWalk
}
return nil
})
return toDownload, err
}
func GetImageIDByImageName(client *docker.Client, image string) (string, error) {
im, err := client.InspectImage(image)
if err != nil {
return "", err
}
return im.ID, nil
}
func ImageExist(client *docker.Client, name, id string) (ex bool, err error) {
image, err := client.InspectImage(name)
if err != nil {
if err == docker.ErrNoSuchImage {
return false, nil
}
return false, err
}
return image.ID == id, nil
}
// equivalentImages executes the provided checks against two docker images, returning true
// if the images are equivalent, and recording a test suite error in any other condition.
func equivalentImages(t *testing.T, c *docker.Client, a, b string, testFilesystem bool, metadataFn metadataFunc, ignoreFns ...ignoreFunc) bool {
imageA, err := c.InspectImage(a)
if err != nil {
t.Errorf("can't get image %q: %v", a, err)
return false
}
imageB, err := c.InspectImage(b)
if err != nil {
t.Errorf("can't get image %q: %v", b, err)
return false
}
if !metadataFn(imageA.Config, imageB.Config) {
t.Errorf("generated image metadata did not match: %s", diff.ObjectDiff(imageA.Config, imageB.Config))
return false
}
// for mutation commands, check the layer diff
if testFilesystem {
differs, onlyA, onlyB, err := compareImageFS(c, a, b)
if err != nil {
t.Errorf("can't calculate FS differences %q: %v", a, err)
return false
}
for k, v := range differs {
if ignoreFuncs(ignoreFns).Ignore(v[0], v[1]) {
delete(differs, k)
continue
}
t.Errorf("%s %s differs: %s", a, k, diff.ObjectDiff(v[0], v[1]))
}
for k, v := range onlyA {
if ignoreFuncs(ignoreFns).Ignore(v, nil) {
delete(onlyA, k)
continue
}
}
for k, v := range onlyB {
if ignoreFuncs(ignoreFns).Ignore(nil, v) {
delete(onlyB, k)
continue
}
}
if len(onlyA)+len(onlyB)+len(differs) > 0 {
t.Errorf("a=%v b=%v diff=%v", onlyA, onlyB, differs)
return false
}
}
return true
}
// BuildAndPushImageOfSizeWithDocker tries to build an image of wanted size and number of layers. It instructs
// Docker daemon directly. Built image is stored as an image stream tag <name>:<tag>. If shouldSucceed is
// false, a push is expected to fail with a denied error. Note the size is only approximate. Resulting image
// size will be different depending on used compression algorithm and metadata overhead.
func BuildAndPushImageOfSizeWithDocker(
oc *exutil.CLI,
dClient *dockerclient.Client,
name, tag string,
size uint64,
numberOfLayers int,
outSink io.Writer,
shouldSucceed bool,
) (imageDigest string, err error) {
registryURL, err := GetDockerRegistryURL(oc)
if err != nil {
return "", err
}
tempDir, err := ioutil.TempDir("", "name-build")
if err != nil {
return "", err
}
dataSize := calculateRoughDataSize(oc.Stdout(), size, numberOfLayers)
lines := make([]string, numberOfLayers+1)
lines[0] = "FROM scratch"
for i := 1; i <= numberOfLayers; i++ {
blobName := fmt.Sprintf("data%d", i)
if err := createRandomBlob(path.Join(tempDir, blobName), dataSize); err != nil {
return "", err
}
lines[i] = fmt.Sprintf("COPY %s /%s", blobName, blobName)
}
if err := ioutil.WriteFile(path.Join(tempDir, "Dockerfile"), []byte(strings.Join(lines, "\n")+"\n"), 0644); err != nil {
return "", err
}
imageName := fmt.Sprintf("%s/%s/%s", registryURL, oc.Namespace(), name)
taggedName := fmt.Sprintf("%s:%s", imageName, tag)
err = dClient.BuildImage(dockerclient.BuildImageOptions{
Name: taggedName,
RmTmpContainer: true,
ForceRmTmpContainer: true,
ContextDir: tempDir,
OutputStream: outSink,
})
if err != nil {
return "", fmt.Errorf("failed to build %q image: %v", taggedName, err)
}
image, err := dClient.InspectImage(taggedName)
if err != nil {
return "", err
}
defer dClient.RemoveImageExtended(image.ID, dockerclient.RemoveImageOptions{Force: true})
digest := ""
if len(image.RepoDigests) == 1 {
digest = image.RepoDigests[0]
}
out, err := oc.Run("whoami").Args("-t").Output()
if err != nil {
return "", err
}
token := strings.TrimSpace(out)
var buf bytes.Buffer
err = dClient.PushImage(dockerclient.PushImageOptions{
Name: imageName,
Tag: tag,
Registry: registryURL,
OutputStream: &buf,
}, dockerclient.AuthConfiguration{
Username: "******",
Password: token,
Email: "*****@*****.**",
ServerAddress: registryURL,
})
out = buf.String()
outSink.Write([]byte(out))
if shouldSucceed {
if err != nil {
return "", fmt.Errorf("Got unexpected push error: %v", err)
}
if len(digest) == 0 {
outSink.Write([]byte("matching digest string\n"))
match := rePushedImageDigest.FindStringSubmatch(out)
if len(match) < 2 {
return "", fmt.Errorf("Failed to parse digest")
}
digest = match[1]
}
return digest, nil
}
if err == nil {
return "", fmt.Errorf("Push unexpectedly succeeded")
}
if !reExpectedDeniedError.MatchString(err.Error()) {
return "", fmt.Errorf("Failed to match expected %q in: %q", reExpectedDeniedError.String(), err.Error())
}
return "", nil
}
func syncImages(client *dockerclient.Client, imageRoot string) error {
logrus.Debugf("Syncing images from %s", imageRoot)
f, err := os.Open(filepath.Join(imageRoot, "images.json"))
if err != nil {
return fmt.Errorf("error opening image json file: %v", err)
}
defer f.Close()
var m tagMap
if err := json.NewDecoder(f).Decode(&m); err != nil {
return fmt.Errorf("error decoding images json: %v", err)
}
allTags := map[string]struct{}{}
neededImages := map[string]struct{}{}
for imageID, tags := range m {
neededImages[imageID] = struct{}{}
for _, t := range tags {
allTags[t] = struct{}{}
}
}
images, err := client.ListImages(dockerclient.ListImagesOptions{})
if err != nil {
return fmt.Errorf("error listing images: %v", err)
}
for _, img := range images {
expectedTags, ok := m[img.ID]
if ok {
delete(neededImages, img.ID)
repoTags := filterRepoTags(img.RepoTags)
logrus.Debugf("Tags for %s: %#v", img.ID, repoTags)
// Sync tags for image ID
removedTags, addedTags := listDiff(repoTags, expectedTags)
for _, t := range addedTags {
if err := tagImage(client, img.ID, t); err != nil {
return err
}
}
for _, t := range removedTags {
// Check if this image tag conflicts with an expected
// tag, in which case force tag will update
if _, ok := allTags[t]; !ok {
logrus.Debugf("Removing tag %s", t)
if err := client.RemoveImage(t); err != nil {
return fmt.Errorf("error removing tag %s: %v", t, err)
}
}
}
} else {
removeOptions := dockerclient.RemoveImageOptions{
Force: true,
}
if err := client.RemoveImageExtended(img.ID, removeOptions); err != nil {
return fmt.Errorf("error moving image %s: %v", img.ID, err)
}
}
}
for imageID := range neededImages {
tags, ok := m[imageID]
if !ok {
return fmt.Errorf("missing image %s in tag map", imageID)
}
_, err := client.InspectImage(imageID)
if err != nil {
tf, err := os.Open(filepath.Join(imageRoot, imageID+".tar"))
if err != nil {
return fmt.Errorf("error opening image tar %s: %v", imageID, err)
}
defer tf.Close()
loadOptions := dockerclient.LoadImageOptions{
InputStream: tf,
}
if err := client.LoadImage(loadOptions); err != nil {
return fmt.Errorf("error loading image %s: %v", imageID, err)
}
}
for _, t := range tags {
if err := tagImage(client, imageID, t); err != nil {
return err
}
}
}
return nil
}
// createAndExtractImage creates a docker container based on the option's image with containerName.
// It will then insepct the container and image and then attempt to extract the image to
// option's destination path. If the destination path is empty it will write to a temp directory
// and update the option's destination path with a /var/tmp directory. /var/tmp is used to
// try and ensure it is a non-in-memory tmpfs.
func (i *defaultImageInspector) createAndExtractImage(client *docker.Client, containerName string) (*docker.Image, error) {
container, err := client.CreateContainer(docker.CreateContainerOptions{
Name: containerName,
Config: &docker.Config{
Image: i.opts.Image,
// For security purpose we don't define any entrypoint and command
Entrypoint: []string{""},
Cmd: []string{""},
},
})
if err != nil {
return nil, fmt.Errorf("Unable to create docker container: %v\n", err)
}
// delete the container when we are done extracting it
defer func() {
client.RemoveContainer(docker.RemoveContainerOptions{
ID: container.ID,
})
}()
containerMetadata, err := client.InspectContainer(container.ID)
if err != nil {
return nil, fmt.Errorf("Unable to get docker container information: %v\n", err)
}
imageMetadata, err := client.InspectImage(containerMetadata.Image)
if err != nil {
return imageMetadata, fmt.Errorf("Unable to get docker image information: %v\n", err)
}
if i.opts.DstPath, err = createOutputDir(i.opts.DstPath, "image-inspector-"); err != nil {
return imageMetadata, err
}
reader, writer := io.Pipe()
// handle closing the reader/writer in the method that creates them
defer writer.Close()
defer reader.Close()
log.Printf("Extracting image %s to %s", i.opts.Image, i.opts.DstPath)
// start the copy function first which will block after the first write while waiting for
// the reader to read.
errorChannel := make(chan error)
go func() {
errorChannel <- client.CopyFromContainer(docker.CopyFromContainerOptions{
Container: container.ID,
OutputStream: writer,
Resource: "/",
})
}()
// block on handling the reads here so we ensure both the write and the reader are finished
// (read waits until an EOF or error occurs).
handleTarStream(reader, i.opts.DstPath)
// capture any error from the copy, ensures both the handleTarStream and CopyFromContainer
// are done.
err = <-errorChannel
if err != nil {
return imageMetadata, fmt.Errorf("Unable to extract container: %v\n", err)
}
return imageMetadata, nil
}
func Inspect(client *docker.Client, name string) (image *docker.Image, err error) {
image, err = client.InspectImage(name)
return
}
func cleanImages(client *docker.Client) {
defer wg.Done()
log.Printf("Img Cleanup: the following images will be locked: %s", *pImageLocked)
log.Println("Img Cleanup: starting image cleanup ...")
for {
// imageIdMap[imageID] = isRemovable
imageIdMap := make(map[string]bool)
// Get the image ID list before the cleanup
images, err := client.ListImages(docker.ListImagesOptions{All: false})
if err != nil {
log.Println("Img Cleanup: cannot get images list", err)
time.Sleep(time.Duration(*pImageCleanInterval+*pImageCleanDelayed) * time.Second)
continue
}
for _, image := range images {
imageIdMap[image.ID] = true
}
// Get the image IDs used by all the containers
containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
log.Println("Img Cleanup: cannot get container list", err)
time.Sleep(time.Duration(*pImageCleanInterval+*pImageCleanDelayed) * time.Second)
continue
} else {
inspect_error := false
for _, container := range containers {
containerInspect, err := client.InspectContainer(container.ID)
if err != nil {
inspect_error = true
log.Println("Img Cleanup: cannot get container inspect", err)
break
}
delete(imageIdMap, containerInspect.Image)
}
if inspect_error {
time.Sleep(time.Duration(*pImageCleanInterval+*pImageCleanDelayed) * time.Second)
continue
}
}
// Get all the locked image ID
if *pImageLocked != "" {
lockedImages := strings.Split(*pImageLocked, ",")
for _, lockedImage := range lockedImages {
imageInspect, err := client.InspectImage(strings.Trim(lockedImage, " "))
if err == nil {
delete(imageIdMap, imageInspect.ID)
}
}
}
// Sleep for the delay time
log.Printf("Img Cleanup: wait %d seconds for the cleaning", *pImageCleanDelayed)
time.Sleep(time.Duration(*pImageCleanDelayed) * time.Second)
// Get the image IDs used by all the containers again after the delay time
containersDelayed, err := client.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
log.Println("Img Cleanup: cannot get container list", err)
time.Sleep(time.Duration(*pImageCleanInterval) * time.Second)
continue
} else {
inspect_error := false
for _, container := range containersDelayed {
containerInspect, err := client.InspectContainer(container.ID)
if err != nil {
inspect_error = true
log.Println("Img Cleanup: cannot get container inspect", err)
break
}
delete(imageIdMap, containerInspect.Image)
}
if inspect_error {
time.Sleep(time.Duration(*pImageCleanInterval) * time.Second)
continue
}
}
// Remove the unused images
counter := 0
for id, removable := range imageIdMap {
if removable {
log.Printf("Img Cleanup: removing image %s", id)
err := client.RemoveImage(id)
if err != nil {
log.Printf("Img Cleanup: %s", err)
}
counter += 1
}
}
log.Printf("Img Cleanup: %d images have been removed", counter)
// Sleep again
log.Printf("Img Cleanup: next cleanup will be start in %d seconds", *pImageCleanInterval)
time.Sleep(time.Duration(*pImageCleanInterval) * time.Second)
}
}