func TestArchiverCancel(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
tmpDir, err := ioutil.TempDir("", "archiver")
ut.AssertEqual(t, nil, err)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Fail()
}
}()
// This will trigger an eventual Cancel().
nonexistent := filepath.Join(tmpDir, "nonexistent")
future1 := a.PushFile("foo", nonexistent, 0)
ut.AssertEqual(t, "foo", future1.DisplayName())
fileName := filepath.Join(tmpDir, "existent")
ut.AssertEqual(t, nil, ioutil.WriteFile(fileName, []byte("foo"), 0600))
future2 := a.PushFile("existent", fileName, 0)
future1.WaitForHashed()
future2.WaitForHashed()
expected := fmt.Errorf("hash(foo) failed: open %s: no such file or directory\n", nonexistent)
if common.IsWindows() {
expected = fmt.Errorf("hash(foo) failed: open %s: The system cannot find the file specified.\n", nonexistent)
}
ut.AssertEqual(t, expected, <-a.Channel())
ut.AssertEqual(t, expected, a.Close())
ut.AssertEqual(t, nil, server.Error())
}
func TestIsolateServerRetryGCSPartial(t *testing.T) {
// GCS upload is teared down in the middle.
t.Parallel()
server := isolatedfake.New()
flaky := &killingMux{server: server, fail: map[string]int{"/fake/cloudstorage": 1}}
ts := httptest.NewServer(flaky)
flaky.ts = ts
defer ts.Close()
client := newIsolateServer(ts.URL, "default-gzip", fastRetry)
large := strings.Repeat("0123456789abcdef", 4096/16)
files := makeItems(large)
expected := map[isolated.HexDigest][]byte{
"5b713884e50a00b44abcee440f2f7d3e2ba701a6": []byte(large),
}
states, err := client.Contains(files.digests)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, len(files.digests), len(states))
for index, state := range states {
err = client.Push(state, bytes.NewReader(files.contents[index]))
ut.AssertEqual(t, nil, err)
}
ut.AssertEqual(t, expected, server.Contents())
states, err = client.Contains(files.digests)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, len(files.digests), len(states))
for _, state := range states {
ut.AssertEqual(t, (*PushState)(nil), state)
}
ut.AssertEqual(t, nil, server.Error())
ut.AssertEqual(t, map[string]int{}, flaky.fail)
}
func testFlaky(t *testing.T, flake string) {
server := isolatedfake.New()
flaky := &failingMux{server: server, fail: map[string]int{flake: 1}}
ts := httptest.NewServer(flaky)
defer ts.Close()
client := newIsolateServer(ts.URL, "default-gzip", fastRetry)
large := strings.Repeat("0123456789abcdef", 4096/16)
files := makeItems("foo", large)
expected := map[isolated.HexDigest][]byte{
"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": []byte("foo"),
"5b713884e50a00b44abcee440f2f7d3e2ba701a6": []byte(large),
}
states, err := client.Contains(files.digests)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, len(files.digests), len(states))
for index, state := range states {
err = client.Push(state, bytes.NewReader(files.contents[index]))
ut.AssertEqual(t, nil, err)
}
ut.AssertEqual(t, expected, server.Contents())
states, err = client.Contains(files.digests)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, len(files.digests), len(states))
for _, state := range states {
ut.AssertEqual(t, (*PushState)(nil), state)
}
ut.AssertEqual(t, nil, server.Error())
ut.AssertEqual(t, map[string]int{}, flaky.fail)
}
func TestIsolateServerCaps(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
client := New(ts.URL, "default-gzip")
caps, err := client.ServerCapabilities()
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, &isolated.ServerCapabilities{"v1"}, caps)
ut.AssertEqual(t, nil, server.Error())
}
func TestArchiverPushSeeked(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
misplaced := bytes.NewReader([]byte("foo"))
_, _ = misplaced.Seek(1, os.SEEK_SET)
future := a.Push("works", misplaced, 0)
future.WaitForHashed()
ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest())
ut.AssertEqual(t, nil, a.Close())
}
func TestArchiverFileHit(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
server.Inject([]byte("foo"))
future := a.Push("foo", bytes.NewReader([]byte("foo")), 0)
future.WaitForHashed()
ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest())
ut.AssertEqual(t, nil, a.Close())
stats := a.Stats()
ut.AssertEqual(t, 1, stats.TotalHits())
ut.AssertEqual(t, 0, stats.TotalMisses())
ut.AssertEqual(t, common.Size(3), stats.TotalBytesHits())
ut.AssertEqual(t, common.Size(0), stats.TotalBytesPushed())
}
func (c *Flags) Parse() error {
if c.ServerURL == "" {
return errors.New("-isolate-server must be specified")
}
if c.ServerURL == "fake" {
ts := httptest.NewServer(isolatedfake.New())
c.ServerURL = ts.URL
} else {
if s, err := lhttp.CheckURL(c.ServerURL); err != nil {
return err
} else {
c.ServerURL = s
}
}
if c.Namespace == "" {
return errors.New("-namespace must be specified")
}
return nil
}
func TestArchiverFile(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
fEmpty, err := ioutil.TempFile("", "archiver")
ut.AssertEqual(t, nil, err)
future1 := a.PushFile(fEmpty.Name(), fEmpty.Name(), 0)
ut.AssertEqual(t, fEmpty.Name(), future1.DisplayName())
fFoo, err := ioutil.TempFile("", "archiver")
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, nil, ioutil.WriteFile(fFoo.Name(), []byte("foo"), 0600))
future2 := a.PushFile(fFoo.Name(), fFoo.Name(), 0)
// Push the same file another time. It'll get linked to the first.
future3 := a.PushFile(fFoo.Name(), fFoo.Name(), 0)
future1.WaitForHashed()
future2.WaitForHashed()
future3.WaitForHashed()
ut.AssertEqual(t, nil, a.Close())
stats := a.Stats()
ut.AssertEqual(t, 0, stats.TotalHits())
// Only 2 lookups, not 3.
ut.AssertEqual(t, 2, stats.TotalMisses())
ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
ut.AssertEqual(t, common.Size(3), stats.TotalBytesPushed())
expected := map[isolated.HexDigest][]byte{
"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": []byte("foo"),
"da39a3ee5e6b4b0d3255bfef95601890afd80709": {},
}
ut.AssertEqual(t, expected, server.Contents())
ut.AssertEqual(t, isolated.HexDigest("da39a3ee5e6b4b0d3255bfef95601890afd80709"), future1.Digest())
ut.AssertEqual(t, nil, future1.Error())
ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future2.Digest())
ut.AssertEqual(t, nil, future2.Error())
ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future3.Digest())
ut.AssertEqual(t, nil, future3.Error())
ut.AssertEqual(t, nil, server.Error())
}
func testNormal(t *testing.T, files items, expected map[isolated.HexDigest][]byte) {
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
client := newIsolateServer(ts.URL, "default-gzip", cantRetry)
states, err := client.Contains(files.digests)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, len(files.digests), len(states))
for index, state := range states {
err = client.Push(state, bytes.NewReader(files.contents[index]))
ut.AssertEqual(t, nil, err)
}
ut.AssertEqual(t, expected, server.Contents())
states, err = client.Contains(files.digests)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, len(files.digests), len(states))
for _, state := range states {
ut.AssertEqual(t, (*PushState)(nil), state)
}
ut.AssertEqual(t, nil, server.Error())
}
func TestArchive(t *testing.T) {
// Create a .isolate file and archive it.
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := archiver.New(isolatedclient.New(ts.URL, "default-gzip"), nil)
// Setup temporary directory.
// /base/bar
// /base/ignored
// /foo/baz.isolate
// /link -> /base/bar
// Result:
// /baz.isolated
tmpDir, err := ioutil.TempDir("", "isolate")
ut.AssertEqual(t, nil, err)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Fail()
}
}()
baseDir := filepath.Join(tmpDir, "base")
fooDir := filepath.Join(tmpDir, "foo")
ut.AssertEqual(t, nil, os.Mkdir(baseDir, 0700))
ut.AssertEqual(t, nil, os.Mkdir(fooDir, 0700))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar"), []byte("foo"), 0600))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "ignored"), []byte("ignored"), 0600))
isolate := `{
'variables': {
'files': [
'../base/',
'../link',
],
},
'conditions': [
['OS=="amiga"', {
'variables': {
'command': ['amiga', '<(EXTRA)'],
},
}],
['OS=="win"', {
'variables': {
'command': ['win'],
},
}],
],
}`
isolatePath := filepath.Join(fooDir, "baz.isolate")
ut.AssertEqual(t, nil, ioutil.WriteFile(isolatePath, []byte(isolate), 0600))
if !common.IsWindows() {
ut.AssertEqual(t, nil, os.Symlink(filepath.Join("base", "bar"), filepath.Join(tmpDir, "link")))
} else {
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(tmpDir, "link"), []byte("no link on Windows"), 0600))
}
opts := &ArchiveOptions{
Isolate: isolatePath,
Isolated: filepath.Join(tmpDir, "baz.isolated"),
Blacklist: common.Strings{"ignored", "*.isolate"},
PathVariables: map[string]string{"VAR": "wonderful"},
ExtraVariables: map[string]string{"EXTRA": "really"},
ConfigVariables: map[string]string{"OS": "amiga"},
}
future := Archive(a, opts)
ut.AssertEqual(t, "baz.isolated", future.DisplayName())
future.WaitForHashed()
ut.AssertEqual(t, nil, future.Error())
ut.AssertEqual(t, nil, a.Close())
mode := 0600
if common.IsWindows() {
mode = 0666
}
// /base/
isolatedDirData := isolated.Isolated{
Algo: "sha-1",
Files: map[string]isolated.File{
filepath.Join("base", "bar"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
},
Version: isolated.IsolatedFormatVersion,
}
encoded, err := json.Marshal(isolatedDirData)
ut.AssertEqual(t, nil, err)
isolatedDirEncoded := string(encoded) + "\n"
isolatedDirHash := isolated.HashBytes([]byte(isolatedDirEncoded))
isolatedData := isolated.Isolated{
Algo: "sha-1",
Command: []string{"amiga", "really"},
Files: map[string]isolated.File{},
Includes: []isolated.HexDigest{isolatedDirHash},
RelativeCwd: "foo",
Version: isolated.IsolatedFormatVersion,
}
if !common.IsWindows() {
isolatedData.Files["link"] = isolated.File{Link: newString(filepath.Join("base", "bar"))}
} else {
isolatedData.Files["link"] = isolated.File{Digest: "12339b9756c2994f85c310d560bc8c142a6b79a1", Mode: newInt(0666), Size: newInt64(18)}
}
encoded, err = json.Marshal(isolatedData)
ut.AssertEqual(t, nil, err)
isolatedEncoded := string(encoded) + "\n"
isolatedHash := isolated.HashBytes([]byte(isolatedEncoded))
expected := map[string]string{
"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": "foo",
string(isolatedDirHash): isolatedDirEncoded,
string(isolatedHash): isolatedEncoded,
}
if common.IsWindows() {
expected["12339b9756c2994f85c310d560bc8c142a6b79a1"] = "no link on Windows"
}
actual := map[string]string{}
for k, v := range server.Contents() {
actual[string(k)] = string(v)
ut.AssertEqualf(t, expected[string(k)], actual[string(k)], "%s: %#v", k, actual[string(k)])
}
ut.AssertEqual(t, expected, actual)
ut.AssertEqual(t, isolatedHash, future.Digest())
stats := a.Stats()
ut.AssertEqual(t, 0, stats.TotalHits())
ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
if !common.IsWindows() {
ut.AssertEqual(t, 3, stats.TotalMisses())
ut.AssertEqual(t, common.Size(3+len(isolatedDirEncoded)+len(isolatedEncoded)), stats.TotalBytesPushed())
} else {
ut.AssertEqual(t, 4, stats.TotalMisses())
ut.AssertEqual(t, common.Size(3+18+len(isolatedDirEncoded)+len(isolatedEncoded)), stats.TotalBytesPushed())
}
ut.AssertEqual(t, nil, server.Error())
digest, err := isolated.HashFile(filepath.Join(tmpDir, "baz.isolated"))
ut.AssertEqual(t, isolated.DigestItem{isolatedHash, false, int64(len(isolatedEncoded))}, digest)
ut.AssertEqual(t, nil, err)
}
func TestPushDirectory(t *testing.T) {
// Uploads a real directory. 2 times the same file.
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
// Setup temporary directory.
tmpDir, err := ioutil.TempDir("", "archiver")
ut.AssertEqual(t, nil, err)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Fail()
}
}()
baseDir := filepath.Join(tmpDir, "base")
ignoredDir := filepath.Join(tmpDir, "ignored1")
ut.AssertEqual(t, nil, os.Mkdir(baseDir, 0700))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar"), []byte("foo"), 0600))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar_dupe"), []byte("foo"), 0600))
if !common.IsWindows() {
ut.AssertEqual(t, nil, os.Symlink("bar", filepath.Join(baseDir, "link")))
}
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "ignored2"), []byte("ignored"), 0600))
ut.AssertEqual(t, nil, os.Mkdir(ignoredDir, 0700))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(ignoredDir, "really"), []byte("ignored"), 0600))
future := PushDirectory(a, tmpDir, "", []string{"ignored1", filepath.Join("*", "ignored2")})
ut.AssertEqual(t, filepath.Base(tmpDir)+".isolated", future.DisplayName())
future.WaitForHashed()
ut.AssertEqual(t, nil, a.Close())
mode := 0600
if common.IsWindows() {
mode = 0666
}
isolatedData := isolated.Isolated{
Algo: "sha-1",
Files: map[string]isolated.File{
filepath.Join("base", "bar"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
filepath.Join("base", "bar_dupe"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
},
Version: isolated.IsolatedFormatVersion,
}
if !common.IsWindows() {
isolatedData.Files[filepath.Join("base", "link")] = isolated.File{Link: newString("bar")}
}
encoded, err := json.Marshal(isolatedData)
ut.AssertEqual(t, nil, err)
isolatedEncoded := string(encoded) + "\n"
isolatedHash := isolated.HashBytes([]byte(isolatedEncoded))
expected := map[string]string{
"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": "foo",
string(isolatedHash): isolatedEncoded,
}
actual := map[string]string{}
for k, v := range server.Contents() {
actual[string(k)] = string(v)
}
ut.AssertEqual(t, expected, actual)
ut.AssertEqual(t, isolatedHash, future.Digest())
stats := a.Stats()
ut.AssertEqual(t, 0, stats.TotalHits())
// There're 3 cache misses even if the same content is looked up twice.
ut.AssertEqual(t, 3, stats.TotalMisses())
ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
ut.AssertEqual(t, common.Size(3+3+len(isolatedEncoded)), stats.TotalBytesPushed())
ut.AssertEqual(t, nil, server.Error())
}