Example #1
0
func (e *ClientExecutor) Archive(src, dst string, allowDecompression, allowDownload bool) (io.Reader, io.Closer, error) {
	var closer closers
	var base string
	var infos []CopyInfo
	var err error
	if isURL(src) {
		if !allowDownload {
			return nil, nil, fmt.Errorf("source can't be a URL")
		}
		infos, base, err = DownloadURL(src, dst)
		if len(base) > 0 {
			closer = append(closer, func() error { return os.RemoveAll(base) })
		}
	} else {
		base = e.Directory
		infos, err = CalcCopyInfo(src, base, allowDecompression, true)
	}
	if err != nil {
		closer.Close()
		return nil, nil, err
	}

	options := archiveOptionsFor(infos, dst, e.Excludes)

	glog.V(4).Infof("Tar of directory %s %#v", base, options)
	rc, err := archive.TarWithOptions(base, options)
	closer = append(closer, rc.Close)
	return rc, closer, err
}
Example #2
0
func createTarStream(srcPath, dockerfilePath string) (io.ReadCloser, error) {
	excludes, err := parseDockerignore(srcPath)
	if err != nil {
		return nil, err
	}

	includes := []string{"."}

	// If .dockerignore mentions .dockerignore or the Dockerfile
	// then make sure we send both files over to the daemon
	// because Dockerfile is, obviously, needed no matter what, and
	// .dockerignore is needed to know if either one needs to be
	// removed.  The deamon will remove them for us, if needed, after it
	// parses the Dockerfile.
	//
	// https://github.com/docker/docker/issues/8330
	//
	forceIncludeFiles := []string{".dockerignore", dockerfilePath}

	for _, includeFile := range forceIncludeFiles {
		if includeFile == "" {
			continue
		}
		keepThem, err := fileutils.Matches(includeFile, excludes)
		if err != nil {
			return nil, fmt.Errorf("cannot match .dockerfile: '%s', error: %s", includeFile, err)
		}
		if keepThem {
			includes = append(includes, includeFile)
		}
	}

	if err := validateContextDirectory(srcPath, excludes); err != nil {
		return nil, err
	}
	tarOpts := &archive.TarOptions{
		ExcludePatterns: excludes,
		IncludeFiles:    includes,
		Compression:     archive.Uncompressed,
		NoLchown:        true,
	}
	return archive.TarWithOptions(srcPath, tarOpts)
}
Example #3
0
func conformanceTester(t *testing.T, c *docker.Client, test conformanceTest, i int, deep bool) {
	dockerfile := test.Dockerfile
	if len(dockerfile) == 0 {
		dockerfile = "Dockerfile"
	}
	tmpDir, err := ioutil.TempDir("", "dockerbuild-conformance-")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	dir := tmpDir
	contextDir := filepath.Join(dir, test.ContextDir)
	dockerfilePath := filepath.Join(dir, dockerfile)

	// clone repo or copy the Dockerfile
	var input string
	switch {
	case len(test.Git) > 0:
		input = test.Git
		cmd := exec.Command("git", "clone", test.Git, dir)
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Errorf("unable to clone %q: %v\n%s", test.Git, err, out)
			return
		}

		if test.PostClone != nil {
			if err := test.PostClone(dir); err != nil {
				t.Errorf("unable to fixup clone: %v", err)
				return
			}
		}

	case len(test.Dockerfile) > 0:
		input = dockerfile
		dockerfilePath = filepath.Join(dir, "Dockerfile")
		if _, err := fileutils.CopyFile(filepath.Join("", dockerfile), dockerfilePath); err != nil {
			t.Fatal(err)
		}
		dockerfile = "Dockerfile"

	default:
		input = filepath.Join(test.ContextDir, dockerfile)
		dockerfilePath = input
		contextDir = test.ContextDir
		dir = test.ContextDir
	}

	// read the dockerfile
	data, err := ioutil.ReadFile(dockerfilePath)
	if err != nil {
		t.Errorf("%d: unable to read Dockerfile %q: %v", i, input, err)
		return
	}
	node, err := parser.Parse(bytes.NewBuffer(data))
	if err != nil {
		t.Errorf("%d: can't parse Dockerfile %q: %v", i, input, err)
		return
	}
	from, err := NewBuilder().From(node)
	if err != nil {
		t.Errorf("%d: can't get base FROM %q: %v", i, input, err)
		return
	}
	nameFormat := "conformance-dockerbuild-%d-%s-%d"

	var toDelete []string
	steps := node.Children
	lastImage := from

	ignoreSmallFileChange := func(a, b *tar.Header) bool {
		if a == nil || b == nil {
			return false
		}
		diff := a.Size - b.Size
		if differOnlyByFileSize(a, b, 10) {
			t.Logf("WARNING: %s differs only in size by %d bytes, probably a timestamp value change", a.Name, diff)
			return true
		}
		return false
	}

	if deep {
		// execute each step on both Docker build and the direct builder, comparing as we
		// go
		fail := false
		for j := range steps {
			testFile := dockerfileWithFrom(lastImage, steps[j:j+1])

			nameDirect := fmt.Sprintf(nameFormat, i, "direct", j)
			nameDocker := fmt.Sprintf(nameFormat, i, "docker", j)

			// run docker build
			if err := ioutil.WriteFile(dockerfilePath, []byte(testFile), 0600); err != nil {
				t.Errorf("%d: unable to update Dockerfile %q: %v", i, dockerfilePath, err)
				break
			}
			in, err := archive.TarWithOptions(dir, &archive.TarOptions{IncludeFiles: []string{"."}})
			if err != nil {
				t.Errorf("%d: unable to generate build context %q: %v", i, dockerfilePath, err)
				break
			}
			out := &bytes.Buffer{}
			if err := c.BuildImage(docker.BuildImageOptions{
				Name:                nameDocker,
				Dockerfile:          dockerfile,
				RmTmpContainer:      true,
				ForceRmTmpContainer: true,
				InputStream:         in,
				OutputStream:        out,
			}); err != nil {
				in.Close()
				t.Errorf("%d: unable to build Docker image %q: %v\n%s", i, test.Git, err, out)
				break
			}
			toDelete = append(toDelete, nameDocker)

			// run direct build
			e := NewClientExecutor(c)
			out = &bytes.Buffer{}
			e.Out, e.ErrOut = out, out
			e.Directory = contextDir
			e.Tag = nameDirect
			if err := e.Build(bytes.NewBufferString(testFile), nil); err != nil {
				t.Errorf("%d: failed to build step %d in dockerfile %q: %s\n%s", i, j, dockerfilePath, steps[j].Original, out)
				break
			}
			toDelete = append(toDelete, nameDirect)

			// only compare filesystem on layers that change the filesystem
			mutation := steps[j].Value == command.Add || steps[j].Value == command.Copy || steps[j].Value == command.Run
			// metadata must be strictly equal
			if !equivalentImages(
				t, c, nameDocker, nameDirect, mutation,
				metadataEqual,
				append(ignoreFuncs{ignoreSmallFileChange}, test.Ignore...)...,
			) {
				t.Errorf("%d: layered Docker build was not equivalent to direct layer image metadata %s", i, input)
				fail = true
			}

			lastImage = nameDocker
		}

		if fail {
			t.Fatalf("%d: Conformance test failed for %s", i, input)
		}

	} else {
		exclude, _ := ParseDockerignore(dir)
		//exclude = append(filtered, ".dockerignore")
		in, err := archive.TarWithOptions(dir, &archive.TarOptions{IncludeFiles: []string{"."}, ExcludePatterns: exclude})
		if err != nil {
			t.Errorf("%d: unable to generate build context %q: %v", i, dockerfilePath, err)
			return
		}
		out := &bytes.Buffer{}
		nameDocker := fmt.Sprintf(nameFormat, i, "docker", 0)
		if err := c.BuildImage(docker.BuildImageOptions{
			Name:                nameDocker,
			Dockerfile:          dockerfile,
			RmTmpContainer:      true,
			ForceRmTmpContainer: true,
			InputStream:         in,
			OutputStream:        out,
		}); err != nil {
			in.Close()
			t.Errorf("%d: unable to build Docker image %q: %v\n%s", i, test.Git, err, out)
			return
		}
		lastImage = nameDocker
		toDelete = append(toDelete, nameDocker)
	}

	// if we ran more than one step, compare the squashed output with the docker build output
	if len(steps) > 1 || !deep {
		nameDirect := fmt.Sprintf(nameFormat, i, "direct", len(steps)-1)
		e := NewClientExecutor(c)
		out := &bytes.Buffer{}
		e.Out, e.ErrOut = out, out
		e.Directory = contextDir
		e.Tag = nameDirect
		if err := e.Build(bytes.NewBuffer(data), nil); err != nil {
			t.Errorf("%d: failed to build complete image in %q: %v\n%s", i, input, err, out)
		} else {
			if !equivalentImages(
				t, c, lastImage, nameDirect, true,
				// metadata should be loosely equivalent, but because we squash and because of limitations
				// in docker commit, there are some differences
				metadataLayerEquivalent,
				append(ignoreFuncs{
					ignoreSmallFileChange,
					// the direct dockerfile contains all steps, the layered image is synthetic from our previous
					// test and so only contains the last layer
					ignoreDockerfileSize(dockerfile),
				}, test.Ignore...)...,
			) {
				t.Errorf("%d: full Docker build was not equivalent to squashed image metadata %s", i, input)
			}
		}
	}

	for _, s := range toDelete {
		c.RemoveImageExtended(s, docker.RemoveImageOptions{Force: true})
	}
}