Beispiel #1
0
func (c *archiveRun) main(a subcommands.Application, args []string) error {
	out := os.Stdout
	prefix := "\n"
	if c.defaultFlags.Quiet {
		out = nil
		prefix = ""
	}
	start := time.Now()
	arch := archiver.New(isolatedclient.New(c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace), out)
	common.CancelOnCtrlC(arch)
	future := isolate.Archive(arch, &c.ArchiveOptions)
	future.WaitForHashed()
	var err error
	if err = future.Error(); err != nil {
		fmt.Printf("%s%s  %s\n", prefix, filepath.Base(c.Isolate), err)
	} else {
		fmt.Printf("%s%s  %s\n", prefix, future.Digest(), filepath.Base(c.Isolate))
	}
	if err2 := arch.Close(); err == nil {
		err = err2
	}
	if !c.defaultFlags.Quiet {
		duration := time.Since(start)
		stats := arch.Stats()
		fmt.Fprintf(os.Stderr, "Hits    : %5d (%s)\n", stats.TotalHits(), stats.TotalBytesHits())
		fmt.Fprintf(os.Stderr, "Misses  : %5d (%s)\n", stats.TotalMisses(), stats.TotalBytesPushed())
		fmt.Fprintf(os.Stderr, "Duration: %s\n", common.Round(duration, time.Millisecond))
	}
	return err
}
Beispiel #2
0
func TestArchiverCancel(t *testing.T) {
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)

	tmpDir, err := ioutil.TempDir("", "archiver")
	ut.AssertEqual(t, nil, err)
	defer func() {
		if err := os.RemoveAll(tmpDir); err != nil {
			t.Fail()
		}
	}()

	// This will trigger an eventual Cancel().
	nonexistent := filepath.Join(tmpDir, "nonexistent")
	future1 := a.PushFile("foo", nonexistent, 0)
	ut.AssertEqual(t, "foo", future1.DisplayName())

	fileName := filepath.Join(tmpDir, "existent")
	ut.AssertEqual(t, nil, ioutil.WriteFile(fileName, []byte("foo"), 0600))
	future2 := a.PushFile("existent", fileName, 0)
	future1.WaitForHashed()
	future2.WaitForHashed()
	expected := fmt.Errorf("hash(foo) failed: open %s: no such file or directory\n", nonexistent)
	if common.IsWindows() {
		expected = fmt.Errorf("hash(foo) failed: open %s: The system cannot find the file specified.\n", nonexistent)
	}
	ut.AssertEqual(t, expected, <-a.Channel())
	ut.AssertEqual(t, expected, a.Close())
	ut.AssertEqual(t, nil, server.Error())
}
Beispiel #3
0
func TestArchiverEmpty(t *testing.T) {
	t.Parallel()
	a := New(isolatedclient.New("https://localhost:1", "default-gzip"), nil)
	stats := a.Stats()
	ut.AssertEqual(t, 0, stats.TotalHits())
	ut.AssertEqual(t, 0, stats.TotalMisses())
	ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
	ut.AssertEqual(t, common.Size(0), stats.TotalBytesPushed())
	ut.AssertEqual(t, nil, a.Close())
}
Beispiel #4
0
func TestArchiverPushSeeked(t *testing.T) {
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
	misplaced := bytes.NewReader([]byte("foo"))
	_, _ = misplaced.Seek(1, os.SEEK_SET)
	future := a.Push("works", misplaced, 0)
	future.WaitForHashed()
	ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest())
	ut.AssertEqual(t, nil, a.Close())
}
Beispiel #5
0
// Test that if the isolate file is not found, the error is properly propagated.
func TestArchiveFileNotFoundReturnsError(t *testing.T) {
	t.Parallel()
	a := archiver.New(isolatedclient.New("http://unused", "default-gzip"), nil)
	opts := &ArchiveOptions{
		Isolate:  "/this-file-does-not-exist",
		Isolated: "/this-file-doesnt-either",
	}
	future := Archive(a, opts)
	future.WaitForHashed()
	err := future.Error()
	ut.AssertEqual(t, true, strings.HasPrefix(err.Error(), "open /this-file-does-not-exist: "))
	closeErr := a.Close()
	ut.AssertEqual(t, true, closeErr != nil)
	ut.AssertEqual(t, true, strings.HasPrefix(closeErr.Error(), "open /this-file-does-not-exist: "))
}
Beispiel #6
0
func TestArchiverFileHit(t *testing.T) {
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
	server.Inject([]byte("foo"))
	future := a.Push("foo", bytes.NewReader([]byte("foo")), 0)
	future.WaitForHashed()
	ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest())
	ut.AssertEqual(t, nil, a.Close())

	stats := a.Stats()
	ut.AssertEqual(t, 1, stats.TotalHits())
	ut.AssertEqual(t, 0, stats.TotalMisses())
	ut.AssertEqual(t, common.Size(3), stats.TotalBytesHits())
	ut.AssertEqual(t, common.Size(0), stats.TotalBytesPushed())
}
Beispiel #7
0
func TestArchiverFile(t *testing.T) {
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)

	fEmpty, err := ioutil.TempFile("", "archiver")
	ut.AssertEqual(t, nil, err)
	future1 := a.PushFile(fEmpty.Name(), fEmpty.Name(), 0)
	ut.AssertEqual(t, fEmpty.Name(), future1.DisplayName())
	fFoo, err := ioutil.TempFile("", "archiver")
	ut.AssertEqual(t, nil, err)
	ut.AssertEqual(t, nil, ioutil.WriteFile(fFoo.Name(), []byte("foo"), 0600))
	future2 := a.PushFile(fFoo.Name(), fFoo.Name(), 0)
	// Push the same file another time. It'll get linked to the first.
	future3 := a.PushFile(fFoo.Name(), fFoo.Name(), 0)
	future1.WaitForHashed()
	future2.WaitForHashed()
	future3.WaitForHashed()
	ut.AssertEqual(t, nil, a.Close())

	stats := a.Stats()
	ut.AssertEqual(t, 0, stats.TotalHits())
	// Only 2 lookups, not 3.
	ut.AssertEqual(t, 2, stats.TotalMisses())
	ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
	ut.AssertEqual(t, common.Size(3), stats.TotalBytesPushed())
	expected := map[isolated.HexDigest][]byte{
		"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": []byte("foo"),
		"da39a3ee5e6b4b0d3255bfef95601890afd80709": {},
	}
	ut.AssertEqual(t, expected, server.Contents())
	ut.AssertEqual(t, isolated.HexDigest("da39a3ee5e6b4b0d3255bfef95601890afd80709"), future1.Digest())
	ut.AssertEqual(t, nil, future1.Error())
	ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future2.Digest())
	ut.AssertEqual(t, nil, future2.Error())
	ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future3.Digest())
	ut.AssertEqual(t, nil, future3.Error())
	ut.AssertEqual(t, nil, server.Error())
}
Beispiel #8
0
func TestArchive(t *testing.T) {
	// Create a .isolate file and archive it.
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := archiver.New(isolatedclient.New(ts.URL, "default-gzip"), nil)

	// Setup temporary directory.
	//   /base/bar
	//   /base/ignored
	//   /foo/baz.isolate
	//   /link -> /base/bar
	// Result:
	//   /baz.isolated
	tmpDir, err := ioutil.TempDir("", "isolate")
	ut.AssertEqual(t, nil, err)
	defer func() {
		if err := os.RemoveAll(tmpDir); err != nil {
			t.Fail()
		}
	}()
	baseDir := filepath.Join(tmpDir, "base")
	fooDir := filepath.Join(tmpDir, "foo")
	ut.AssertEqual(t, nil, os.Mkdir(baseDir, 0700))
	ut.AssertEqual(t, nil, os.Mkdir(fooDir, 0700))
	ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar"), []byte("foo"), 0600))
	ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "ignored"), []byte("ignored"), 0600))
	isolate := `{
		'variables': {
			'files': [
				'../base/',
				'../link',
			],
		},
		'conditions': [
			['OS=="amiga"', {
				'variables': {
					'command': ['amiga', '<(EXTRA)'],
				},
			}],
			['OS=="win"', {
				'variables': {
					'command': ['win'],
				},
			}],
		],
	}`
	isolatePath := filepath.Join(fooDir, "baz.isolate")
	ut.AssertEqual(t, nil, ioutil.WriteFile(isolatePath, []byte(isolate), 0600))
	if !common.IsWindows() {
		ut.AssertEqual(t, nil, os.Symlink(filepath.Join("base", "bar"), filepath.Join(tmpDir, "link")))
	} else {
		ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(tmpDir, "link"), []byte("no link on Windows"), 0600))
	}
	opts := &ArchiveOptions{
		Isolate:         isolatePath,
		Isolated:        filepath.Join(tmpDir, "baz.isolated"),
		Blacklist:       common.Strings{"ignored", "*.isolate"},
		PathVariables:   map[string]string{"VAR": "wonderful"},
		ExtraVariables:  map[string]string{"EXTRA": "really"},
		ConfigVariables: map[string]string{"OS": "amiga"},
	}
	future := Archive(a, opts)
	ut.AssertEqual(t, "baz.isolated", future.DisplayName())
	future.WaitForHashed()
	ut.AssertEqual(t, nil, future.Error())
	ut.AssertEqual(t, nil, a.Close())

	mode := 0600
	if common.IsWindows() {
		mode = 0666
	}
	//   /base/
	isolatedDirData := isolated.Isolated{
		Algo: "sha-1",
		Files: map[string]isolated.File{
			filepath.Join("base", "bar"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
		},
		Version: isolated.IsolatedFormatVersion,
	}
	encoded, err := json.Marshal(isolatedDirData)
	ut.AssertEqual(t, nil, err)
	isolatedDirEncoded := string(encoded) + "\n"
	isolatedDirHash := isolated.HashBytes([]byte(isolatedDirEncoded))

	isolatedData := isolated.Isolated{
		Algo:        "sha-1",
		Command:     []string{"amiga", "really"},
		Files:       map[string]isolated.File{},
		Includes:    []isolated.HexDigest{isolatedDirHash},
		RelativeCwd: "foo",
		Version:     isolated.IsolatedFormatVersion,
	}
	if !common.IsWindows() {
		isolatedData.Files["link"] = isolated.File{Link: newString(filepath.Join("base", "bar"))}
	} else {
		isolatedData.Files["link"] = isolated.File{Digest: "12339b9756c2994f85c310d560bc8c142a6b79a1", Mode: newInt(0666), Size: newInt64(18)}
	}
	encoded, err = json.Marshal(isolatedData)
	ut.AssertEqual(t, nil, err)
	isolatedEncoded := string(encoded) + "\n"
	isolatedHash := isolated.HashBytes([]byte(isolatedEncoded))

	expected := map[string]string{
		"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": "foo",
		string(isolatedDirHash):                    isolatedDirEncoded,
		string(isolatedHash):                       isolatedEncoded,
	}
	if common.IsWindows() {
		expected["12339b9756c2994f85c310d560bc8c142a6b79a1"] = "no link on Windows"
	}
	actual := map[string]string{}
	for k, v := range server.Contents() {
		actual[string(k)] = string(v)
		ut.AssertEqualf(t, expected[string(k)], actual[string(k)], "%s: %#v", k, actual[string(k)])
	}
	ut.AssertEqual(t, expected, actual)
	ut.AssertEqual(t, isolatedHash, future.Digest())

	stats := a.Stats()
	ut.AssertEqual(t, 0, stats.TotalHits())
	ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
	if !common.IsWindows() {
		ut.AssertEqual(t, 3, stats.TotalMisses())
		ut.AssertEqual(t, common.Size(3+len(isolatedDirEncoded)+len(isolatedEncoded)), stats.TotalBytesPushed())
	} else {
		ut.AssertEqual(t, 4, stats.TotalMisses())
		ut.AssertEqual(t, common.Size(3+18+len(isolatedDirEncoded)+len(isolatedEncoded)), stats.TotalBytesPushed())
	}

	ut.AssertEqual(t, nil, server.Error())
	digest, err := isolated.HashFile(filepath.Join(tmpDir, "baz.isolated"))
	ut.AssertEqual(t, isolated.DigestItem{isolatedHash, false, int64(len(isolatedEncoded))}, digest)
	ut.AssertEqual(t, nil, err)
}
Beispiel #9
0
func TestPushDirectory(t *testing.T) {
	// Uploads a real directory. 2 times the same file.
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)

	// Setup temporary directory.
	tmpDir, err := ioutil.TempDir("", "archiver")
	ut.AssertEqual(t, nil, err)
	defer func() {
		if err := os.RemoveAll(tmpDir); err != nil {
			t.Fail()
		}
	}()
	baseDir := filepath.Join(tmpDir, "base")
	ignoredDir := filepath.Join(tmpDir, "ignored1")
	ut.AssertEqual(t, nil, os.Mkdir(baseDir, 0700))
	ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar"), []byte("foo"), 0600))
	ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar_dupe"), []byte("foo"), 0600))
	if !common.IsWindows() {
		ut.AssertEqual(t, nil, os.Symlink("bar", filepath.Join(baseDir, "link")))
	}
	ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "ignored2"), []byte("ignored"), 0600))
	ut.AssertEqual(t, nil, os.Mkdir(ignoredDir, 0700))
	ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(ignoredDir, "really"), []byte("ignored"), 0600))

	future := PushDirectory(a, tmpDir, "", []string{"ignored1", filepath.Join("*", "ignored2")})
	ut.AssertEqual(t, filepath.Base(tmpDir)+".isolated", future.DisplayName())
	future.WaitForHashed()
	ut.AssertEqual(t, nil, a.Close())

	mode := 0600
	if common.IsWindows() {
		mode = 0666
	}
	isolatedData := isolated.Isolated{
		Algo: "sha-1",
		Files: map[string]isolated.File{
			filepath.Join("base", "bar"):      {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
			filepath.Join("base", "bar_dupe"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
		},
		Version: isolated.IsolatedFormatVersion,
	}
	if !common.IsWindows() {
		isolatedData.Files[filepath.Join("base", "link")] = isolated.File{Link: newString("bar")}
	}
	encoded, err := json.Marshal(isolatedData)
	ut.AssertEqual(t, nil, err)
	isolatedEncoded := string(encoded) + "\n"
	isolatedHash := isolated.HashBytes([]byte(isolatedEncoded))

	expected := map[string]string{
		"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": "foo",
		string(isolatedHash):                       isolatedEncoded,
	}
	actual := map[string]string{}
	for k, v := range server.Contents() {
		actual[string(k)] = string(v)
	}
	ut.AssertEqual(t, expected, actual)
	ut.AssertEqual(t, isolatedHash, future.Digest())

	stats := a.Stats()
	ut.AssertEqual(t, 0, stats.TotalHits())
	// There're 3 cache misses even if the same content is looked up twice.
	ut.AssertEqual(t, 3, stats.TotalMisses())
	ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
	ut.AssertEqual(t, common.Size(3+3+len(isolatedEncoded)), stats.TotalBytesPushed())

	ut.AssertEqual(t, nil, server.Error())
}
Beispiel #10
0
func (c *batchArchiveRun) main(a subcommands.Application, args []string) error {
	out := os.Stdout
	prefix := "\n"
	if c.defaultFlags.Quiet {
		out = nil
		prefix = ""
	}
	start := time.Now()
	arch := archiver.New(isolatedclient.New(c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace), out)
	common.CancelOnCtrlC(arch)
	type tmp struct {
		name   string
		future archiver.Future
	}
	items := make(chan *tmp, len(args))
	var wg sync.WaitGroup
	for _, arg := range args {
		wg.Add(1)
		go func(genJsonPath string) {
			defer wg.Done()
			data := &struct {
				Args    []string
				Dir     string
				Version int
			}{}
			if err := common.ReadJSONFile(genJsonPath, data); err != nil {
				arch.Cancel(err)
				return
			}
			if data.Version != isolate.IsolatedGenJSONVersion {
				arch.Cancel(fmt.Errorf("invalid version %d in %s", data.Version, genJsonPath))
				return
			}
			if !common.IsDirectory(data.Dir) {
				arch.Cancel(fmt.Errorf("invalid dir %s in %s", data.Dir, genJsonPath))
				return
			}
			opts, err := parseArchiveCMD(data.Args, data.Dir)
			if err != nil {
				arch.Cancel(fmt.Errorf("invalid archive command in %s: %s", genJsonPath, err))
				return
			}
			name := filepath.Base(opts.Isolated)
			// Strip the extension if there is one.
			if dotIndex := strings.LastIndex(name, "."); dotIndex != -1 {
				name = name[0:dotIndex]
			}
			items <- &tmp{name, isolate.Archive(arch, opts)}
		}(arg)
	}
	go func() {
		wg.Wait()
		close(items)
	}()

	data := map[string]isolated.HexDigest{}
	for item := range items {
		item.future.WaitForHashed()
		if item.future.Error() == nil {
			data[item.name] = item.future.Digest()
			fmt.Printf("%s%s  %s\n", prefix, item.future.Digest(), item.name)
		} else {
			fmt.Fprintf(os.Stderr, "%s%s  %s\n", prefix, item.name, item.future.Error())
		}
	}
	err := arch.Close()
	duration := time.Since(start)
	// Only write the file once upload is confirmed.
	if err == nil && c.dumpJSON != "" {
		err = common.WriteJSONFile(c.dumpJSON, data)
	}
	if !c.defaultFlags.Quiet {
		stats := arch.Stats()
		fmt.Fprintf(os.Stderr, "Hits    : %5d (%s)\n", stats.TotalHits(), stats.TotalBytesHits())
		fmt.Fprintf(os.Stderr, "Misses  : %5d (%s)\n", stats.TotalMisses(), stats.TotalBytesPushed())
		fmt.Fprintf(os.Stderr, "Duration: %s\n", common.Round(duration, time.Millisecond))
	}
	return err
}