func TestArchiverCancel(t *testing.T) { t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := New(isolatedclient.New(ts.URL, "default-gzip"), nil) tmpDir, err := ioutil.TempDir("", "archiver") ut.AssertEqual(t, nil, err) defer func() { if err := os.RemoveAll(tmpDir); err != nil { t.Fail() } }() // This will trigger an eventual Cancel(). nonexistent := filepath.Join(tmpDir, "nonexistent") future1 := a.PushFile("foo", nonexistent, 0) ut.AssertEqual(t, "foo", future1.DisplayName()) fileName := filepath.Join(tmpDir, "existent") ut.AssertEqual(t, nil, ioutil.WriteFile(fileName, []byte("foo"), 0600)) future2 := a.PushFile("existent", fileName, 0) future1.WaitForHashed() future2.WaitForHashed() expected := fmt.Errorf("hash(foo) failed: open %s: no such file or directory\n", nonexistent) if common.IsWindows() { expected = fmt.Errorf("hash(foo) failed: open %s: The system cannot find the file specified.\n", nonexistent) } ut.AssertEqual(t, expected, <-a.Channel()) ut.AssertEqual(t, expected, a.Close()) ut.AssertEqual(t, nil, server.Error()) }
func TestIsolateServerRetryGCSPartial(t *testing.T) { // GCS upload is teared down in the middle. t.Parallel() server := isolatedfake.New() flaky := &killingMux{server: server, fail: map[string]int{"/fake/cloudstorage": 1}} ts := httptest.NewServer(flaky) flaky.ts = ts defer ts.Close() client := newIsolateServer(ts.URL, "default-gzip", fastRetry) large := strings.Repeat("0123456789abcdef", 4096/16) files := makeItems(large) expected := map[isolated.HexDigest][]byte{ "5b713884e50a00b44abcee440f2f7d3e2ba701a6": []byte(large), } states, err := client.Contains(files.digests) ut.AssertEqual(t, nil, err) ut.AssertEqual(t, len(files.digests), len(states)) for index, state := range states { err = client.Push(state, bytes.NewReader(files.contents[index])) ut.AssertEqual(t, nil, err) } ut.AssertEqual(t, expected, server.Contents()) states, err = client.Contains(files.digests) ut.AssertEqual(t, nil, err) ut.AssertEqual(t, len(files.digests), len(states)) for _, state := range states { ut.AssertEqual(t, (*PushState)(nil), state) } ut.AssertEqual(t, nil, server.Error()) ut.AssertEqual(t, map[string]int{}, flaky.fail) }
func testFlaky(t *testing.T, flake string) { server := isolatedfake.New() flaky := &failingMux{server: server, fail: map[string]int{flake: 1}} ts := httptest.NewServer(flaky) defer ts.Close() client := newIsolateServer(ts.URL, "default-gzip", fastRetry) large := strings.Repeat("0123456789abcdef", 4096/16) files := makeItems("foo", large) expected := map[isolated.HexDigest][]byte{ "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": []byte("foo"), "5b713884e50a00b44abcee440f2f7d3e2ba701a6": []byte(large), } states, err := client.Contains(files.digests) ut.AssertEqual(t, nil, err) ut.AssertEqual(t, len(files.digests), len(states)) for index, state := range states { err = client.Push(state, bytes.NewReader(files.contents[index])) ut.AssertEqual(t, nil, err) } ut.AssertEqual(t, expected, server.Contents()) states, err = client.Contains(files.digests) ut.AssertEqual(t, nil, err) ut.AssertEqual(t, len(files.digests), len(states)) for _, state := range states { ut.AssertEqual(t, (*PushState)(nil), state) } ut.AssertEqual(t, nil, server.Error()) ut.AssertEqual(t, map[string]int{}, flaky.fail) }
func TestIsolateServerCaps(t *testing.T) { t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() client := New(ts.URL, "default-gzip") caps, err := client.ServerCapabilities() ut.AssertEqual(t, nil, err) ut.AssertEqual(t, &isolated.ServerCapabilities{"v1"}, caps) ut.AssertEqual(t, nil, server.Error()) }
func TestArchiverPushSeeked(t *testing.T) { t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := New(isolatedclient.New(ts.URL, "default-gzip"), nil) misplaced := bytes.NewReader([]byte("foo")) _, _ = misplaced.Seek(1, os.SEEK_SET) future := a.Push("works", misplaced, 0) future.WaitForHashed() ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest()) ut.AssertEqual(t, nil, a.Close()) }
func TestArchiverFileHit(t *testing.T) { t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := New(isolatedclient.New(ts.URL, "default-gzip"), nil) server.Inject([]byte("foo")) future := a.Push("foo", bytes.NewReader([]byte("foo")), 0) future.WaitForHashed() ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest()) ut.AssertEqual(t, nil, a.Close()) stats := a.Stats() ut.AssertEqual(t, 1, stats.TotalHits()) ut.AssertEqual(t, 0, stats.TotalMisses()) ut.AssertEqual(t, common.Size(3), stats.TotalBytesHits()) ut.AssertEqual(t, common.Size(0), stats.TotalBytesPushed()) }
func (c *Flags) Parse() error { if c.ServerURL == "" { return errors.New("-isolate-server must be specified") } if c.ServerURL == "fake" { ts := httptest.NewServer(isolatedfake.New()) c.ServerURL = ts.URL } else { if s, err := lhttp.CheckURL(c.ServerURL); err != nil { return err } else { c.ServerURL = s } } if c.Namespace == "" { return errors.New("-namespace must be specified") } return nil }
func TestArchiverFile(t *testing.T) { t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := New(isolatedclient.New(ts.URL, "default-gzip"), nil) fEmpty, err := ioutil.TempFile("", "archiver") ut.AssertEqual(t, nil, err) future1 := a.PushFile(fEmpty.Name(), fEmpty.Name(), 0) ut.AssertEqual(t, fEmpty.Name(), future1.DisplayName()) fFoo, err := ioutil.TempFile("", "archiver") ut.AssertEqual(t, nil, err) ut.AssertEqual(t, nil, ioutil.WriteFile(fFoo.Name(), []byte("foo"), 0600)) future2 := a.PushFile(fFoo.Name(), fFoo.Name(), 0) // Push the same file another time. It'll get linked to the first. future3 := a.PushFile(fFoo.Name(), fFoo.Name(), 0) future1.WaitForHashed() future2.WaitForHashed() future3.WaitForHashed() ut.AssertEqual(t, nil, a.Close()) stats := a.Stats() ut.AssertEqual(t, 0, stats.TotalHits()) // Only 2 lookups, not 3. ut.AssertEqual(t, 2, stats.TotalMisses()) ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits()) ut.AssertEqual(t, common.Size(3), stats.TotalBytesPushed()) expected := map[isolated.HexDigest][]byte{ "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": []byte("foo"), "da39a3ee5e6b4b0d3255bfef95601890afd80709": {}, } ut.AssertEqual(t, expected, server.Contents()) ut.AssertEqual(t, isolated.HexDigest("da39a3ee5e6b4b0d3255bfef95601890afd80709"), future1.Digest()) ut.AssertEqual(t, nil, future1.Error()) ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future2.Digest()) ut.AssertEqual(t, nil, future2.Error()) ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future3.Digest()) ut.AssertEqual(t, nil, future3.Error()) ut.AssertEqual(t, nil, server.Error()) }
func testNormal(t *testing.T, files items, expected map[isolated.HexDigest][]byte) { server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() client := newIsolateServer(ts.URL, "default-gzip", cantRetry) states, err := client.Contains(files.digests) ut.AssertEqual(t, nil, err) ut.AssertEqual(t, len(files.digests), len(states)) for index, state := range states { err = client.Push(state, bytes.NewReader(files.contents[index])) ut.AssertEqual(t, nil, err) } ut.AssertEqual(t, expected, server.Contents()) states, err = client.Contains(files.digests) ut.AssertEqual(t, nil, err) ut.AssertEqual(t, len(files.digests), len(states)) for _, state := range states { ut.AssertEqual(t, (*PushState)(nil), state) } ut.AssertEqual(t, nil, server.Error()) }
func TestArchive(t *testing.T) { // Create a .isolate file and archive it. t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := archiver.New(isolatedclient.New(ts.URL, "default-gzip"), nil) // Setup temporary directory. // /base/bar // /base/ignored // /foo/baz.isolate // /link -> /base/bar // Result: // /baz.isolated tmpDir, err := ioutil.TempDir("", "isolate") ut.AssertEqual(t, nil, err) defer func() { if err := os.RemoveAll(tmpDir); err != nil { t.Fail() } }() baseDir := filepath.Join(tmpDir, "base") fooDir := filepath.Join(tmpDir, "foo") ut.AssertEqual(t, nil, os.Mkdir(baseDir, 0700)) ut.AssertEqual(t, nil, os.Mkdir(fooDir, 0700)) ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar"), []byte("foo"), 0600)) ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "ignored"), []byte("ignored"), 0600)) isolate := `{ 'variables': { 'files': [ '../base/', '../link', ], }, 'conditions': [ ['OS=="amiga"', { 'variables': { 'command': ['amiga', '<(EXTRA)'], }, }], ['OS=="win"', { 'variables': { 'command': ['win'], }, }], ], }` isolatePath := filepath.Join(fooDir, "baz.isolate") ut.AssertEqual(t, nil, ioutil.WriteFile(isolatePath, []byte(isolate), 0600)) if !common.IsWindows() { ut.AssertEqual(t, nil, os.Symlink(filepath.Join("base", "bar"), filepath.Join(tmpDir, "link"))) } else { ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(tmpDir, "link"), []byte("no link on Windows"), 0600)) } opts := &ArchiveOptions{ Isolate: isolatePath, Isolated: filepath.Join(tmpDir, "baz.isolated"), Blacklist: common.Strings{"ignored", "*.isolate"}, PathVariables: map[string]string{"VAR": "wonderful"}, ExtraVariables: map[string]string{"EXTRA": "really"}, ConfigVariables: map[string]string{"OS": "amiga"}, } future := Archive(a, opts) ut.AssertEqual(t, "baz.isolated", future.DisplayName()) future.WaitForHashed() ut.AssertEqual(t, nil, future.Error()) ut.AssertEqual(t, nil, a.Close()) mode := 0600 if common.IsWindows() { mode = 0666 } // /base/ isolatedDirData := isolated.Isolated{ Algo: "sha-1", Files: map[string]isolated.File{ filepath.Join("base", "bar"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)}, }, Version: isolated.IsolatedFormatVersion, } encoded, err := json.Marshal(isolatedDirData) ut.AssertEqual(t, nil, err) isolatedDirEncoded := string(encoded) + "\n" isolatedDirHash := isolated.HashBytes([]byte(isolatedDirEncoded)) isolatedData := isolated.Isolated{ Algo: "sha-1", Command: []string{"amiga", "really"}, Files: map[string]isolated.File{}, Includes: []isolated.HexDigest{isolatedDirHash}, RelativeCwd: "foo", Version: isolated.IsolatedFormatVersion, } if !common.IsWindows() { isolatedData.Files["link"] = isolated.File{Link: newString(filepath.Join("base", "bar"))} } else { isolatedData.Files["link"] = isolated.File{Digest: "12339b9756c2994f85c310d560bc8c142a6b79a1", Mode: newInt(0666), Size: newInt64(18)} } encoded, err = json.Marshal(isolatedData) ut.AssertEqual(t, nil, err) isolatedEncoded := string(encoded) + "\n" isolatedHash := isolated.HashBytes([]byte(isolatedEncoded)) expected := map[string]string{ "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": "foo", string(isolatedDirHash): isolatedDirEncoded, string(isolatedHash): isolatedEncoded, } if common.IsWindows() { expected["12339b9756c2994f85c310d560bc8c142a6b79a1"] = "no link on Windows" } actual := map[string]string{} for k, v := range server.Contents() { actual[string(k)] = string(v) ut.AssertEqualf(t, expected[string(k)], actual[string(k)], "%s: %#v", k, actual[string(k)]) } ut.AssertEqual(t, expected, actual) ut.AssertEqual(t, isolatedHash, future.Digest()) stats := a.Stats() ut.AssertEqual(t, 0, stats.TotalHits()) ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits()) if !common.IsWindows() { ut.AssertEqual(t, 3, stats.TotalMisses()) ut.AssertEqual(t, common.Size(3+len(isolatedDirEncoded)+len(isolatedEncoded)), stats.TotalBytesPushed()) } else { ut.AssertEqual(t, 4, stats.TotalMisses()) ut.AssertEqual(t, common.Size(3+18+len(isolatedDirEncoded)+len(isolatedEncoded)), stats.TotalBytesPushed()) } ut.AssertEqual(t, nil, server.Error()) digest, err := isolated.HashFile(filepath.Join(tmpDir, "baz.isolated")) ut.AssertEqual(t, isolated.DigestItem{isolatedHash, false, int64(len(isolatedEncoded))}, digest) ut.AssertEqual(t, nil, err) }
func TestPushDirectory(t *testing.T) { // Uploads a real directory. 2 times the same file. t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := New(isolatedclient.New(ts.URL, "default-gzip"), nil) // Setup temporary directory. tmpDir, err := ioutil.TempDir("", "archiver") ut.AssertEqual(t, nil, err) defer func() { if err := os.RemoveAll(tmpDir); err != nil { t.Fail() } }() baseDir := filepath.Join(tmpDir, "base") ignoredDir := filepath.Join(tmpDir, "ignored1") ut.AssertEqual(t, nil, os.Mkdir(baseDir, 0700)) ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar"), []byte("foo"), 0600)) ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar_dupe"), []byte("foo"), 0600)) if !common.IsWindows() { ut.AssertEqual(t, nil, os.Symlink("bar", filepath.Join(baseDir, "link"))) } ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "ignored2"), []byte("ignored"), 0600)) ut.AssertEqual(t, nil, os.Mkdir(ignoredDir, 0700)) ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(ignoredDir, "really"), []byte("ignored"), 0600)) future := PushDirectory(a, tmpDir, "", []string{"ignored1", filepath.Join("*", "ignored2")}) ut.AssertEqual(t, filepath.Base(tmpDir)+".isolated", future.DisplayName()) future.WaitForHashed() ut.AssertEqual(t, nil, a.Close()) mode := 0600 if common.IsWindows() { mode = 0666 } isolatedData := isolated.Isolated{ Algo: "sha-1", Files: map[string]isolated.File{ filepath.Join("base", "bar"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)}, filepath.Join("base", "bar_dupe"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)}, }, Version: isolated.IsolatedFormatVersion, } if !common.IsWindows() { isolatedData.Files[filepath.Join("base", "link")] = isolated.File{Link: newString("bar")} } encoded, err := json.Marshal(isolatedData) ut.AssertEqual(t, nil, err) isolatedEncoded := string(encoded) + "\n" isolatedHash := isolated.HashBytes([]byte(isolatedEncoded)) expected := map[string]string{ "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": "foo", string(isolatedHash): isolatedEncoded, } actual := map[string]string{} for k, v := range server.Contents() { actual[string(k)] = string(v) } ut.AssertEqual(t, expected, actual) ut.AssertEqual(t, isolatedHash, future.Digest()) stats := a.Stats() ut.AssertEqual(t, 0, stats.TotalHits()) // There're 3 cache misses even if the same content is looked up twice. ut.AssertEqual(t, 3, stats.TotalMisses()) ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits()) ut.AssertEqual(t, common.Size(3+3+len(isolatedEncoded)), stats.TotalBytesPushed()) ut.AssertEqual(t, nil, server.Error()) }