func TestArchiverCancel(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
tmpDir, err := ioutil.TempDir("", "archiver")
ut.AssertEqual(t, nil, err)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Fail()
}
}()
// This will trigger an eventual Cancel().
nonexistent := filepath.Join(tmpDir, "nonexistent")
future1 := a.PushFile("foo", nonexistent, 0)
ut.AssertEqual(t, "foo", future1.DisplayName())
fileName := filepath.Join(tmpDir, "existent")
ut.AssertEqual(t, nil, ioutil.WriteFile(fileName, []byte("foo"), 0600))
future2 := a.PushFile("existent", fileName, 0)
future1.WaitForHashed()
future2.WaitForHashed()
expected := fmt.Errorf("hash(foo) failed: open %s: no such file or directory\n", nonexistent)
if common.IsWindows() {
expected = fmt.Errorf("hash(foo) failed: open %s: The system cannot find the file specified.\n", nonexistent)
}
ut.AssertEqual(t, expected, <-a.Channel())
ut.AssertEqual(t, expected, a.Close())
ut.AssertEqual(t, nil, server.Error())
}
func TestLoadIsolateAsConfig(t *testing.T) {
t.Parallel()
root := "/dir"
if common.IsWindows() {
root = "x:\\dir"
}
isolate, err := LoadIsolateAsConfig(root, []byte(sampleIsolateData))
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, []string{"OS", "bit"}, isolate.ConfigVariables)
}
// Init initializes with non-nil values.
func (a *ArchiveOptions) Init() {
a.Blacklist = common.Strings{}
a.PathVariables = map[string]string{}
if common.IsWindows() {
a.PathVariables["EXECUTABLE_SUFFIX"] = ".exe"
} else {
a.PathVariables["EXECUTABLE_SUFFIX"] = ""
}
a.ExtraVariables = map[string]string{}
a.ConfigVariables = map[string]string{}
}
func loadIncludedIsolate(isolateDir, include string) (*Configs, error) {
if filepath.IsAbs(include) {
return nil, fmt.Errorf("failed to load configuration; absolute include path %s", include)
}
includedIsolate := filepath.Clean(filepath.Join(isolateDir, include))
if common.IsWindows() && (strings.ToLower(includedIsolate)[0] != strings.ToLower(isolateDir)[0]) {
return nil, errors.New("can't reference a .isolate file from another drive")
}
content, err := ioutil.ReadFile(includedIsolate)
if err != nil {
return nil, err
}
return LoadIsolateAsConfig(filepath.Dir(includedIsolate), content)
}
func TestLoadIsolateForConfig(t *testing.T) {
t.Parallel()
// Case linux64, matches first condition.
root := "/dir"
if common.IsWindows() {
root = "x:\\dir"
}
vars := map[string]string{"bit": "64", "OS": "linux"}
cmd, deps, ro, dir, err := LoadIsolateForConfig(root, []byte(sampleIsolateData), vars)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, root, dir)
ut.AssertEqual(t, NotSet, ro) // first condition has no read_only specified.
ut.AssertEqual(t, []string{"python", "64linuxOrWin"}, cmd)
ut.AssertEqual(t, []string{"64linuxOrWin", filepath.Join("<(PRODUCT_DIR)", "unittest<(EXECUTABLE_SUFFIX)")}, deps)
// Case win64, matches only first condition.
vars = map[string]string{"bit": "64", "OS": "win"}
cmd, deps, ro, dir, err = LoadIsolateForConfig(root, []byte(sampleIsolateData), vars)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, root, dir)
ut.AssertEqual(t, NotSet, ro) // first condition has no read_only specified.
ut.AssertEqual(t, []string{"python", "64linuxOrWin"}, cmd)
ut.AssertEqual(t, []string{
"64linuxOrWin",
filepath.Join("<(PRODUCT_DIR)", "unittest<(EXECUTABLE_SUFFIX)"),
}, deps)
// Case mac64, matches only second condition.
vars = map[string]string{"bit": "64", "OS": "mac"}
cmd, deps, ro, dir, err = LoadIsolateForConfig(root, []byte(sampleIsolateData), vars)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, root, dir)
ut.AssertEqual(t, DirsReadOnly, ro) // second condition has read_only 2.
ut.AssertEqual(t, []string{"python", "32orMac64"}, cmd)
ut.AssertEqual(t, []string{}, deps)
// Case win32, both first and second condition match.
vars = map[string]string{"bit": "32", "OS": "win"}
cmd, deps, ro, dir, err = LoadIsolateForConfig(root, []byte(sampleIsolateData), vars)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, root, dir)
ut.AssertEqual(t, DirsReadOnly, ro) // first condition no read_only, but second has 2.
ut.AssertEqual(t, []string{"python", "32orMac64"}, cmd)
ut.AssertEqual(t, []string{
"64linuxOrWin",
filepath.Join("<(PRODUCT_DIR)", "unittest<(EXECUTABLE_SUFFIX)"),
}, deps)
}
func TestLoadIsolateForConfigMissingVars(t *testing.T) {
t.Parallel()
isoData := []byte(sampleIsolateData)
root := "/dir"
if common.IsWindows() {
root = "x:\\dir"
}
_, _, _, _, err := LoadIsolateForConfig(root, isoData, nil)
ut.AssertEqual(t, true, err != nil)
ut.AssertEqualf(t, true, strings.Contains(err.Error(), "variables were missing"), "%s", err)
ut.AssertEqualf(t, true, strings.Contains(err.Error(), "bit"), "%s", err)
ut.AssertEqualf(t, true, strings.Contains(err.Error(), "OS"), "%s", err)
_, _, _, _, err = LoadIsolateForConfig(root, isoData, map[string]string{"bit": "32"})
ut.AssertEqual(t, true, err != nil)
ut.AssertEqualf(t, true, strings.Contains(err.Error(), "variables were missing"), "%s", err)
ut.AssertEqualf(t, true, strings.Contains(err.Error(), "OS"), "%s", err)
}
func TestWalkInexistent(t *testing.T) {
ch := make(chan *walkItem)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
defer close(ch)
walk("inexistent_directory", nil, ch)
}()
item := <-ch
err := errors.New("walk(inexistent_directory): lstat inexistent_directory: no such file or directory")
if common.IsWindows() {
err = errors.New("walk(inexistent_directory): GetFileAttributesEx inexistent_directory: The system cannot find the file specified.")
}
ut.AssertEqual(t, &walkItem{err: err}, item)
item, ok := <-ch
ut.AssertEqual(t, (*walkItem)(nil), item)
ut.AssertEqual(t, false, ok)
wg.Wait()
}
func TestArchiveCMDParsing(t *testing.T) {
t.Parallel()
args := []string{
"--isolated", "../biz/bar.isolated",
"--isolate", "../boz/bar.isolate",
"--path-variable", "DEPTH", "../..",
"--path-variable", "PRODUCT_DIR", "../../out/Release",
"--extra-variable", "version_full=42.0.2284.0",
"--config-variable", "OS=linux",
}
root := absToOS("e:", "/tmp/bar")
opts, err := parseArchiveCMD(args, root)
base := filepath.Dir(root)
ut.AssertEqual(t, filepath.Join(base, "boz", "bar.isolate"), opts.Isolate)
ut.AssertEqual(t, filepath.Join(base, "biz", "bar.isolated"), opts.Isolated)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, opts.ConfigVariables, stringmapflag.Value{"OS": "linux"})
if common.IsWindows() {
ut.AssertEqual(t, opts.PathVariables, stringmapflag.Value{"PRODUCT_DIR": "../../out/Release", "EXECUTABLE_SUFFIX": ".exe", "DEPTH": "../.."})
} else {
ut.AssertEqual(t, opts.PathVariables, stringmapflag.Value{"PRODUCT_DIR": "../../out/Release", "EXECUTABLE_SUFFIX": "", "DEPTH": "../.."})
}
ut.AssertEqual(t, opts.ExtraVariables, stringmapflag.Value{"version_full": "42.0.2284.0"})
}
// absToOS converts a POSIX path to OS specific format.
func absToOS(p string) string {
if common.IsWindows() {
return "e:" + strings.Replace(p, "/", "\\", -1)
}
return p
}
// union merges two config settings together into a new instance.
//
// A new instance is not created and self or rhs is returned if the other
// object is the empty object.
//
// self has priority over rhs for Command. Use the same IsolateDir as the
// one having a Command.
//
// Dependencies listed in rhs are patch adjusted ONLY if they don't start with
// a path variable, e.g. the characters '<('.
func (lhs *ConfigSettings) union(rhs *ConfigSettings) (*ConfigSettings, error) {
// When an object has IsolateDir == "", it means it is the empty object.
if lhs.IsolateDir == "" {
return rhs, nil
}
if rhs.IsolateDir == "" {
return lhs, nil
}
if common.IsWindows() && strings.ToLower(lhs.IsolateDir)[0] != strings.ToLower(rhs.IsolateDir)[0] {
return nil, errors.New("All .isolate files must be on same drive")
}
// Takes the difference between the two isolateDir. Note that while
// isolateDir is in native path case, all other references are in posix.
useRHS := false
var command []string
if len(lhs.Command) > 0 {
useRHS = false
command = lhs.Command
} else if len(rhs.Command) > 0 {
useRHS = true
command = rhs.Command
} else {
// If self doesn't define any file, use rhs.
useRHS = len(lhs.Files) == 0
}
readOnly := rhs.ReadOnly
if lhs.ReadOnly != NotSet {
readOnly = lhs.ReadOnly
}
lRelCwd, rRelCwd := lhs.IsolateDir, rhs.IsolateDir
lFiles, rFiles := lhs.Files, rhs.Files
if useRHS {
// Rebase files in rhs.
lRelCwd, rRelCwd = rhs.IsolateDir, lhs.IsolateDir
lFiles, rFiles = rhs.Files, lhs.Files
}
rebasePath, err := filepath.Rel(lRelCwd, rRelCwd)
if err != nil {
return nil, err
}
rebasePath = strings.Replace(rebasePath, osPathSeparator, "/", -1)
filesSet := map[string]bool{}
for _, f := range lFiles {
filesSet[f] = true
}
for _, f := range rFiles {
// Rebase item.
if !(strings.HasPrefix(f, "<(") || rebasePath == ".") {
// paths are posix here.
trailingSlash := strings.HasSuffix(f, "/")
f = path.Join(rebasePath, f)
if trailingSlash {
f += "/"
}
}
filesSet[f] = true
}
// Remove duplicates.
files := make([]string, 0, len(filesSet))
for f := range filesSet {
files = append(files, f)
}
sort.Strings(files)
return &ConfigSettings{files, command, readOnly, lRelCwd}, nil
}
func TestArchive(t *testing.T) {
// Create a .isolate file and archive it.
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := archiver.New(isolatedclient.New(ts.URL, "default-gzip"), nil)
// Setup temporary directory.
// /base/bar
// /base/ignored
// /foo/baz.isolate
// /link -> /base/bar
// Result:
// /baz.isolated
tmpDir, err := ioutil.TempDir("", "isolate")
ut.AssertEqual(t, nil, err)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Fail()
}
}()
baseDir := filepath.Join(tmpDir, "base")
fooDir := filepath.Join(tmpDir, "foo")
ut.AssertEqual(t, nil, os.Mkdir(baseDir, 0700))
ut.AssertEqual(t, nil, os.Mkdir(fooDir, 0700))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar"), []byte("foo"), 0600))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "ignored"), []byte("ignored"), 0600))
isolate := `{
'variables': {
'files': [
'../base/',
'../link',
],
},
'conditions': [
['OS=="amiga"', {
'variables': {
'command': ['amiga', '<(EXTRA)'],
},
}],
['OS=="win"', {
'variables': {
'command': ['win'],
},
}],
],
}`
isolatePath := filepath.Join(fooDir, "baz.isolate")
ut.AssertEqual(t, nil, ioutil.WriteFile(isolatePath, []byte(isolate), 0600))
if !common.IsWindows() {
ut.AssertEqual(t, nil, os.Symlink(filepath.Join("base", "bar"), filepath.Join(tmpDir, "link")))
} else {
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(tmpDir, "link"), []byte("no link on Windows"), 0600))
}
opts := &ArchiveOptions{
Isolate: isolatePath,
Isolated: filepath.Join(tmpDir, "baz.isolated"),
Blacklist: common.Strings{"ignored", "*.isolate"},
PathVariables: map[string]string{"VAR": "wonderful"},
ExtraVariables: map[string]string{"EXTRA": "really"},
ConfigVariables: map[string]string{"OS": "amiga"},
}
future := Archive(a, opts)
ut.AssertEqual(t, "baz.isolated", future.DisplayName())
future.WaitForHashed()
ut.AssertEqual(t, nil, future.Error())
ut.AssertEqual(t, nil, a.Close())
mode := 0600
if common.IsWindows() {
mode = 0666
}
// /base/
isolatedDirData := isolated.Isolated{
Algo: "sha-1",
Files: map[string]isolated.File{
filepath.Join("base", "bar"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
},
Version: isolated.IsolatedFormatVersion,
}
encoded, err := json.Marshal(isolatedDirData)
ut.AssertEqual(t, nil, err)
isolatedDirEncoded := string(encoded) + "\n"
isolatedDirHash := isolated.HashBytes([]byte(isolatedDirEncoded))
isolatedData := isolated.Isolated{
Algo: "sha-1",
Command: []string{"amiga", "really"},
Files: map[string]isolated.File{},
Includes: []isolated.HexDigest{isolatedDirHash},
RelativeCwd: "foo",
Version: isolated.IsolatedFormatVersion,
}
if !common.IsWindows() {
isolatedData.Files["link"] = isolated.File{Link: newString(filepath.Join("base", "bar"))}
} else {
isolatedData.Files["link"] = isolated.File{Digest: "12339b9756c2994f85c310d560bc8c142a6b79a1", Mode: newInt(0666), Size: newInt64(18)}
}
encoded, err = json.Marshal(isolatedData)
ut.AssertEqual(t, nil, err)
isolatedEncoded := string(encoded) + "\n"
isolatedHash := isolated.HashBytes([]byte(isolatedEncoded))
expected := map[string]string{
"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": "foo",
string(isolatedDirHash): isolatedDirEncoded,
string(isolatedHash): isolatedEncoded,
}
if common.IsWindows() {
expected["12339b9756c2994f85c310d560bc8c142a6b79a1"] = "no link on Windows"
}
actual := map[string]string{}
for k, v := range server.Contents() {
actual[string(k)] = string(v)
ut.AssertEqualf(t, expected[string(k)], actual[string(k)], "%s: %#v", k, actual[string(k)])
}
ut.AssertEqual(t, expected, actual)
ut.AssertEqual(t, isolatedHash, future.Digest())
stats := a.Stats()
ut.AssertEqual(t, 0, stats.TotalHits())
ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
if !common.IsWindows() {
ut.AssertEqual(t, 3, stats.TotalMisses())
ut.AssertEqual(t, common.Size(3+len(isolatedDirEncoded)+len(isolatedEncoded)), stats.TotalBytesPushed())
} else {
ut.AssertEqual(t, 4, stats.TotalMisses())
ut.AssertEqual(t, common.Size(3+18+len(isolatedDirEncoded)+len(isolatedEncoded)), stats.TotalBytesPushed())
}
ut.AssertEqual(t, nil, server.Error())
digest, err := isolated.HashFile(filepath.Join(tmpDir, "baz.isolated"))
ut.AssertEqual(t, isolated.DigestItem{isolatedHash, false, int64(len(isolatedEncoded))}, digest)
ut.AssertEqual(t, nil, err)
}
func TestPushDirectory(t *testing.T) {
// Uploads a real directory. 2 times the same file.
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
// Setup temporary directory.
tmpDir, err := ioutil.TempDir("", "archiver")
ut.AssertEqual(t, nil, err)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Fail()
}
}()
baseDir := filepath.Join(tmpDir, "base")
ignoredDir := filepath.Join(tmpDir, "ignored1")
ut.AssertEqual(t, nil, os.Mkdir(baseDir, 0700))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar"), []byte("foo"), 0600))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "bar_dupe"), []byte("foo"), 0600))
if !common.IsWindows() {
ut.AssertEqual(t, nil, os.Symlink("bar", filepath.Join(baseDir, "link")))
}
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(baseDir, "ignored2"), []byte("ignored"), 0600))
ut.AssertEqual(t, nil, os.Mkdir(ignoredDir, 0700))
ut.AssertEqual(t, nil, ioutil.WriteFile(filepath.Join(ignoredDir, "really"), []byte("ignored"), 0600))
future := PushDirectory(a, tmpDir, "", []string{"ignored1", filepath.Join("*", "ignored2")})
ut.AssertEqual(t, filepath.Base(tmpDir)+".isolated", future.DisplayName())
future.WaitForHashed()
ut.AssertEqual(t, nil, a.Close())
mode := 0600
if common.IsWindows() {
mode = 0666
}
isolatedData := isolated.Isolated{
Algo: "sha-1",
Files: map[string]isolated.File{
filepath.Join("base", "bar"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
filepath.Join("base", "bar_dupe"): {Digest: "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", Mode: newInt(mode), Size: newInt64(3)},
},
Version: isolated.IsolatedFormatVersion,
}
if !common.IsWindows() {
isolatedData.Files[filepath.Join("base", "link")] = isolated.File{Link: newString("bar")}
}
encoded, err := json.Marshal(isolatedData)
ut.AssertEqual(t, nil, err)
isolatedEncoded := string(encoded) + "\n"
isolatedHash := isolated.HashBytes([]byte(isolatedEncoded))
expected := map[string]string{
"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": "foo",
string(isolatedHash): isolatedEncoded,
}
actual := map[string]string{}
for k, v := range server.Contents() {
actual[string(k)] = string(v)
}
ut.AssertEqual(t, expected, actual)
ut.AssertEqual(t, isolatedHash, future.Digest())
stats := a.Stats()
ut.AssertEqual(t, 0, stats.TotalHits())
// There're 3 cache misses even if the same content is looked up twice.
ut.AssertEqual(t, 3, stats.TotalMisses())
ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
ut.AssertEqual(t, common.Size(3+3+len(isolatedEncoded)), stats.TotalBytesPushed())
ut.AssertEqual(t, nil, server.Error())
}
// absToOS converts a POSIX path to OS specific format.
func absToOS(drive, p string) string {
if common.IsWindows() {
return drive + strings.Replace(p, "/", "\\", -1)
}
return p
}
