// writeIndex calls x.Write with the index's backing file.
func writeIndex(fs rwvfs.FileSystem, name string, x persistedIndex) (err error) {
vlog.Printf("%s: writing index...", name)
f, err := fs.Create(fmt.Sprintf(indexFilename, name))
if err != nil {
return err
}
defer func() {
err2 := f.Close()
if err == nil {
err = err2
}
}()
w := gzip.NewWriter(f)
if err := x.Write(w); err != nil {
return err
}
if err := w.Flush(); err != nil {
return err
}
if err := w.Close(); err != nil {
return err
}
vlog.Printf("%s: done writing index.", name)
return nil
}
func isFile(fs rwvfs.FileSystem, file string) bool {
fi, err := fs.Stat(file)
if err != nil {
return false
}
return fi.Mode().IsRegular()
}
// readIndex calls x.Read with the index's backing file.
func readIndex(fs rwvfs.FileSystem, name string, x persistedIndex) (err error) {
vlog.Printf("%s: reading index...", name)
var f vfs.ReadSeekCloser
f, err = fs.Open(fmt.Sprintf(indexFilename, name))
if err != nil {
vlog.Printf("%s: failed to read index: %s.", name, err)
if os.IsNotExist(err) {
return &errIndexNotExist{name: name, err: err}
}
return err
}
defer func() {
err2 := f.Close()
if err == nil {
err = err2
}
}()
r, err := gzip.NewReader(f)
if err != nil {
return err
}
if err := x.Read(r); err != nil {
return err
}
if err := r.Close(); err != nil {
return err
}
vlog.Printf("%s: done reading index.", name)
return nil
}
func PathDoesNotExist(t *testing.T, label string, fs rwvfs.FileSystem, path string) {
fi, err := fs.Stat(path)
if err != nil && !os.IsNotExist(err) {
t.Errorf("%s: Stat(%q): want os.IsNotExist-satisfying error, got %q", label, path, err)
} else if err == nil {
t.Errorf("%s: Stat(%q): want file to not exist, got existing file with FileInfo %+v", label, path, fi)
}
}
func testWrite(t *testing.T, fs rwvfs.FileSystem, path string) {
label := fmt.Sprintf("%T", fs)
w, err := fs.Create(path)
if err != nil {
t.Fatalf("%s: WriterOpen: %s", label, err)
}
input := []byte("qux")
_, err = w.Write(input)
if err != nil {
t.Fatalf("%s: Write: %s", label, err)
}
err = w.Close()
if err != nil {
t.Fatalf("%s: w.Close: %s", label, err)
}
var r io.ReadCloser
r, err = fs.Open(path)
if err != nil {
t.Fatalf("%s: Open: %s", label, err)
}
var output []byte
output, err = ioutil.ReadAll(r)
if err != nil {
t.Fatalf("%s: ReadAll: %s", label, err)
}
if !bytes.Equal(output, input) {
t.Errorf("%s: got output %q, want %q", label, output, input)
}
r, err = fs.Open(path)
if err != nil {
t.Fatalf("%s: Open: %s", label, err)
}
output, err = ioutil.ReadAll(r)
if err != nil {
t.Fatalf("%s: ReadAll: %s", label, err)
}
if !bytes.Equal(output, input) {
t.Errorf("%s: got output %q, want %q", label, output, input)
}
if err := fs.Remove(path); err != nil {
t.Errorf("%s: Remove(%q): %s", label, path, err)
}
time.Sleep(time.Second)
fi, err := fs.Stat(path)
if err != nil && !os.IsNotExist(err) {
t.Errorf("%s: Stat(%q): want os.IsNotExist-satisfying error, got %q", label, path, err)
} else if err == nil {
t.Errorf("%s: Stat(%q): want file to not exist, got existing file with FileInfo %+v", label, path, fi)
}
}
func IsFile(t *testing.T, label string, fs rwvfs.FileSystem, path string) {
fi, err := fs.Stat(path)
if err != nil {
t.Fatalf("%s: Stat(%q): %s", label, path, err)
}
if !fi.Mode().IsRegular() {
t.Errorf("%s: got fs.Stat(%q) Mode().IsRegular() == false, want true", label, path)
}
}
func createFile(t *testing.T, fs rwvfs.FileSystem, path string, contents []byte) {
w, err := fs.Create(path)
if err != nil {
t.Fatalf("Create(%s): %s", path, err)
}
if _, err := w.Write(contents); err != nil {
t.Fatalf("Write(%s): %s", path, err)
}
if err := w.Close(); err != nil {
t.Fatalf("w.Close(): %s", err)
}
}
func testCreate(t *testing.T, u rwvfs.FileSystem, path string, m map[string]string) {
w, err := u.Create(path)
if err != nil {
t.Fatal(err)
}
w.Write([]byte("test"))
w.Close()
if got, want := m[path], "test"; got != want {
t.Errorf(`Create: got %v, want %v`, got, want)
}
}
func IsDir(t *testing.T, label string, fs rwvfs.FileSystem, path string) {
fi, err := fs.Stat(path)
if err != nil {
t.Fatalf("%s: Stat(%q): %s", label, path, err)
}
if fi == nil {
t.Fatalf("%s: FileInfo (%q) == nil", label, path)
}
if !fi.IsDir() {
t.Errorf("%s: got fs.Stat(%q) IsDir() == false, want true", label, path)
}
}
func setCreateParentDirs(fs rwvfs.FileSystem) {
type createParents interface {
CreateParentDirs(bool)
}
if fs, ok := fs.(createParents); ok {
fs.CreateParentDirs(true)
}
}
func testGlob(t *testing.T, fs rwvfs.FileSystem) {
label := fmt.Sprintf("%T", fs)
files := []string{"x/y/0.txt", "x/y/1.txt", "x/2.txt"}
for _, file := range files {
err := rwvfs.MkdirAll(fs, filepath.Dir(file))
if err != nil {
t.Fatalf("%s: MkdirAll: %s", label, err)
}
w, err := fs.Create(file)
if err != nil {
t.Errorf("%s: Create(%q): %s", label, file, err)
return
}
_, err = w.Write([]byte("x"))
if err != nil {
t.Fatal(err)
}
err = w.Close()
if err != nil {
t.Fatal(err)
}
}
globTests := []struct {
prefix string
pattern string
matches []string
}{
{"", "x/y/*.txt", []string{"x/y/0.txt", "x/y/1.txt"}},
{"x/y", "x/y/*.txt", []string{"x/y/0.txt", "x/y/1.txt"}},
{"", "x/*", []string{"x/y", "x/2.txt"}},
}
for _, test := range globTests {
matches, err := rwvfs.Glob(walkableFileSystem{fs}, test.prefix, test.pattern)
if err != nil {
t.Errorf("%s: Glob(prefix=%q, pattern=%q): %s", label, test.prefix, test.pattern, err)
continue
}
sort.Strings(test.matches)
sort.Strings(matches)
if !reflect.DeepEqual(matches, test.matches) {
t.Errorf("%s: Glob(prefix=%q, pattern=%q): got %v, want %v", label, test.prefix, test.pattern, matches, test.matches)
}
}
}
func uploadFile(local vfs.FileSystem, remote rwvfs.FileSystem, path string, fi os.FileInfo, dryRun bool) error {
kb := float64(fi.Size()) / 1024
if GlobalOpt.Verbose || dryRun {
log.Printf("Uploading %s (%.1fkb)", path, kb)
}
if dryRun {
return nil
}
lf, err := local.Open(path)
if err != nil {
return err
}
if err := rwvfs.MkdirAll(remote, filepath.Dir(path)); err != nil {
return err
}
rf, err := remote.Create(path)
if err != nil {
return err
}
defer func() {
if err := rf.Close(); err != nil {
log.Println("Error closing after error:", err)
}
}()
if _, err := io.Copy(rf, lf); err != nil {
return err
}
if err := rf.Close(); err != nil {
return err
}
if GlobalOpt.Verbose {
log.Printf("Uploaded %s (%.1fkb)", path, kb)
}
return nil
}
func fetchFile(remote vfs.FileSystem, local rwvfs.FileSystem, path string, fi os.FileInfo, dryRun bool) error {
kb := float64(fi.Size()) / 1024
if GlobalOpt.Verbose || dryRun {
log.Printf("Fetching %s (%.1fkb)", path, kb)
}
if dryRun {
return nil
}
if err := rwvfs.MkdirAll(local, filepath.Dir(path)); err != nil {
return err
}
rf, err := remote.Open(path)
if err != nil {
return fmt.Errorf("remote file: %s", err)
}
defer rf.Close()
lf, err := local.Create(path)
if err != nil {
return fmt.Errorf("local file: %s", err)
}
defer lf.Close()
if _, err := io.Copy(lf, rf); err != nil {
return fmt.Errorf("copy from remote to local: %s", err)
}
if GlobalOpt.Verbose {
log.Printf("Fetched %s (%.1fkb)", path, kb)
}
if err := lf.Close(); err != nil {
return fmt.Errorf("local file: %s", err)
}
return nil
}
// rangeReader calls ioutil.ReadAll on the given byte range [start, n). It uses
// optimizations for different kinds of VFSs.
func rangeReader(fs rwvfs.FileSystem, name string, f io.ReadSeeker, start, n int64) (io.Reader, error) {
if fs, ok := fs.(rwvfs.FetcherOpener); ok {
// Clone f so we can parallelize it.
var err error
f, err = fs.OpenFetcher(name)
if err != nil {
return nil, err
}
if err := f.(rwvfs.Fetcher).Fetch(start, start+n); err != nil {
return nil, err
}
}
if _, err := f.Seek(start, 0); err != nil {
return nil, err
}
return f, nil
}
// statIndex calls fs.Stat on the index's backing file or dir.
func statIndex(fs rwvfs.FileSystem, name string) (os.FileInfo, error) {
return fs.Stat(fmt.Sprintf(indexFilename, name))
}
func testStat(t *testing.T, fs rwvfs.FileSystem, path string) {
label := fmt.Sprintf("Stat %T", fs)
cases := []struct {
parent, child string
checkDirs []string
}{
{pathpkg.Join(path, "p"), pathpkg.Join(path, "p/c"), nil},
{pathpkg.Join(path, "."), pathpkg.Join(path, "c"), nil},
{pathpkg.Join(path, "p1/p2"), pathpkg.Join(path, "p1/p2/p3/c"), []string{"p1", "p1/p2/p3"}},
{pathpkg.Join(path, "p1"), pathpkg.Join(path, "p1/p2/p3/c"), []string{"p1/p2", "p1/p2/p3"}},
}
// Clean out bucket.
for _, x := range cases {
removeFile(t, fs, x.parent)
removeFile(t, fs, x.child)
}
removeFile(t, fs, path)
if path != "." {
if _, err := fs.Stat(path); !os.IsNotExist(err) {
t.Fatalf("%s: Stat(%s): got error %v, want os.IsNotExist-satisfying", label, path, err)
}
}
if _, err := fs.Stat(path + "/z"); !os.IsNotExist(err) {
t.Fatalf("%s: Stat(%s): got error %v, want os.IsNotExist-satisfying", label, path+"/z", err)
}
// Not sure of the best way to treat S3 keys that are
// delimiter-prefixes of other keys, since they can either be like
// dirs or files. But let's just choose a way and add a test so we
// can change the behavior easily later.
for _, x := range cases {
t.Logf("# parent %q, child %q", x.parent, x.child)
createFile(t, fs, x.parent, []byte("x"))
createFile(t, fs, x.child, []byte("x"))
parentFI, err := fs.Stat(x.parent)
if err != nil {
t.Fatalf("%s: Stat(%s): %s", label, x.parent, err)
}
if !parentFI.Mode().IsDir() {
t.Fatalf("%s: Stat(%s) got Mode().IsDir() == false, want true", label, x.parent)
}
childFI, err := fs.Stat(x.child)
if err != nil {
t.Fatalf("%s: Stat(%s): %s", label, x.child, err)
}
if !childFI.Mode().IsRegular() {
t.Fatalf("%s: Stat(%s) got Mode().IsRegular() == false, want true", label, x.child)
}
// Should not exist.
doesntExist := pathpkg.Join(x.child, "doesntexist")
if _, err := fs.Stat(doesntExist); !os.IsNotExist(err) {
t.Fatalf("%s: Stat(%s): got error %v, want os.IsNotExist-satisfying", label, doesntExist, err)
}
for _, dir := range x.checkDirs {
dir = pathpkg.Join(path, dir)
fi, err := fs.Stat(dir)
if err != nil {
t.Fatalf("%s: Stat(%s): %s", label, dir, err)
}
if !fi.Mode().IsDir() {
t.Fatalf("%s: Stat(%s): not dir, want dir", label, dir)
}
}
if x.parent != "." {
// Check that the parent file can be opened like a file.
f, err := fs.Open(x.parent)
if err != nil {
t.Fatalf("%s: Open(%s): %s", label, x.parent, err)
}
f.Close()
}
// Clean up
if err := fs.Remove(x.parent); err != nil {
t.Errorf("%s: Remove(%q): %s", label, x.parent, err)
}
if err := fs.Remove(x.child); err != nil {
t.Errorf("%s: Remove(%q): %s", label, x.child, err)
}
}
}
func removeFile(t *testing.T, fs rwvfs.FileSystem, path string) {
if err := fs.Remove(path); err != nil {
t.Fatalf("removeFile(%q): %s", path, err)
}
}
// openFetcher calls fs.OpenFetcher if it implemented the
// FetcherOpener interface; otherwise it calls fs.Open.
func openFetcherOrOpen(fs rwvfs.FileSystem, name string) (vfs.ReadSeekCloser, error) {
if fo, ok := fs.(rwvfs.FetcherOpener); ok {
return fo.OpenFetcher(name)
}
return fs.Open(name)
}
func Mkdir(t *testing.T, fs rwvfs.FileSystem) {
label := fmt.Sprintf("%T", fs)
if strings.Contains(label, "subFS") {
if err := fs.Mkdir("/"); err != nil && !os.IsExist(err) {
t.Fatalf("%s: subFS Mkdir(/): %s", label, err)
}
}
if strings.Contains(label, "mapFS") {
if err := fs.Mkdir("/"); err != nil && !os.IsExist(err) {
t.Fatalf("%s: mapFS Mkdir(/): %s", label, err)
}
}
fi, err := fs.Stat(".")
if err != nil {
t.Fatalf("%s: Stat(.): %s", label, err)
}
if !fi.Mode().IsDir() {
t.Fatalf("%s: got Stat(.) FileMode %o, want IsDir", label, fi.Mode())
}
fi, err = fs.Stat("/")
if err != nil {
t.Fatalf("%s: Stat(/): %s", label, err)
}
if !fi.Mode().IsDir() {
t.Fatalf("%s: got Stat(/) FileMode %o, want IsDir", label, fi.Mode())
}
if _, err := fs.ReadDir("."); err != nil {
t.Fatalf("%s: ReadDir(.): %s", label, err)
}
if _, err := fs.ReadDir("/"); err != nil {
t.Fatalf("%s: ReadDir(/): %s", label, err)
}
fis, err := fs.ReadDir("/")
if err != nil {
t.Fatalf("%s: ReadDir(/): %s", label, err)
}
if len(fis) != 0 {
t.Fatalf("%s: ReadDir(/): got %d file infos (%v), want none (is it including .?)", label, len(fis), fis)
}
err = fs.Mkdir("dir0")
if err != nil {
t.Fatalf("%s: Mkdir(dir0): %s", label, err)
}
IsDir(t, label, fs, "dir0")
IsDir(t, label, fs, "/dir0")
err = fs.Mkdir("/dir1")
if err != nil {
t.Fatalf("%s: Mkdir(/dir1): %s", label, err)
}
IsDir(t, label, fs, "dir1")
IsDir(t, label, fs, "/dir1")
err = fs.Mkdir("/dir1")
if !os.IsExist(err) {
t.Errorf("%s: Mkdir(/dir1) again: got err %v, want os.IsExist-satisfying error", label, err)
}
err = fs.Mkdir("/parent-doesnt-exist/dir2")
if !os.IsNotExist(err) {
t.Errorf("%s: Mkdir(/parent-doesnt-exist/dir2): got error %v, want os.IsNotExist-satisfying error", label, err)
}
err = fs.Remove("/dir1")
if err != nil {
t.Errorf("%s: Remove(/dir1): %s", label, err)
}
PathDoesNotExist(t, label, fs, "/dir1")
}
func testOpen(t *testing.T, fs rwvfs.FileSystem) {
const path = "testOpen"
var buf bytes.Buffer
for i := uint8(0); i < 255; i++ {
for j := uint8(0); j < 255; j++ {
buf.Write([]byte{i, j})
}
}
fullData := []byte(base64.StdEncoding.EncodeToString(buf.Bytes()))[10:]
fullLen := int64(len(fullData))
createFile(t, fs, path, fullData)
{
// Full reads.
f, err := fs.Open(path)
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, fullData) {
t.Errorf("full read: got %q, want %q", b, fullData)
}
}
{
// Partial reads.
rrt := &rangeRecordingTransport{}
fs.(*S3FS).config.Client = &http.Client{Transport: rrt}
var f vfs.ReadSeekCloser
cases := [][2]int64{
{0, 0},
{0, 1},
{0, 2},
{1, 1},
{0, 3},
{1, 3},
{2, 3},
{0, 2},
{0, 3},
{3, 4},
{0, fullLen / 2},
{1, fullLen / 2},
{fullLen / 3, fullLen / 2},
{0, fullLen - 1},
{1, fullLen - 1},
{fullLen / 2, fullLen/2 + 1333},
{fullLen / 2, fullLen/2 + 1},
{fullLen / 2, fullLen/2 + 2},
{fullLen / 2, fullLen / 2},
{fullLen - 10, fullLen - 1},
}
for _, autofetch := range []bool{false, true} {
for _, reuse := range []bool{false, true} {
for i, c := range cases {
if !reuse || i == 0 {
var err error
f, err = fs.(rwvfs.FetcherOpener).OpenFetcher(path)
if err != nil {
t.Fatal(err)
}
}
f.(interface {
SetAutofetch(bool)
}).SetAutofetch(true)
rrt.reset()
start, end := c[0], c[1]
label := fmt.Sprintf("range %d-%d (autofetch=%v, reuse=%v)", start, end, autofetch, reuse)
fetchEnd := end
if autofetch {
// Short fetch.
fetchEnd = (start + end) / 2
if fetchEnd < start {
fetchEnd = end
}
}
if err := f.(rwvfs.Fetcher).Fetch(start, fetchEnd); err != nil {
t.Error(err)
continue
}
n, err := f.Seek(start, 0)
if err != nil {
t.Errorf("%s: %s", label, err)
continue
}
if n != start {
t.Errorf("got post-Seek offset %d, want %d", n, start)
}
b, err := ioutil.ReadAll(io.LimitReader(f, end-start))
if err != nil {
t.Errorf("%s: ReadAll: %s", label, err)
continue
}
trunc := func(b []byte) string {
if len(b) > 75 {
return string(b[:75]) + "..." + string(b[len(b)-5:]) + fmt.Sprintf(" (%d bytes total)", len(b))
}
return string(b)
}
if want := fullData[start:end]; !bytes.Equal(b, want) {
t.Errorf("%s: full read: got %q, want %q", label, trunc(b), trunc(want))
continue
}
if start != end && !reuse {
if len(rrt.readRanges) == 0 {
t.Errorf("%s: no read ranges, want range %d-%d", label, start, end)
}
}
if !autofetch {
if err := rrt.checkOnlyReadRange(start, end); err != nil {
t.Errorf("%s: %s", label, err)
}
}
if !reuse || i == len(cases)-1 {
if err := f.Close(); err != nil {
t.Fatal(err)
}
}
}
}
}
}
}
func Write(t *testing.T, fs rwvfs.FileSystem) {
label := fmt.Sprintf("%T", fs)
fs.Mkdir("/foo")
defer fs.Remove("/foo")
path := "/foo/bar"
w, err := fs.Create(path)
if err != nil {
t.Fatalf("%s: WriterOpen: %s", label, err)
}
input := []byte("qux")
_, err = w.Write(input)
if err != nil {
t.Fatalf("%s: Write: %s", label, err)
}
err = w.Close()
if err != nil {
t.Fatalf("%s: w.Close: %s", label, err)
}
var r io.ReadCloser
r, err = fs.Open(path)
if err != nil {
t.Fatalf("%s: Open: %s", label, err)
}
var output []byte
output, err = ioutil.ReadAll(r)
if err != nil {
t.Fatalf("%s: ReadAll: %s", label, err)
}
err = r.Close()
if err != nil {
t.Fatalf("%s: r.Close: %s", label, err)
}
if !bytes.Equal(output, input) {
t.Errorf("%s: got output %q, want %q", label, output, input)
}
IsFile(t, label, fs, path)
IsDir(t, label, fs, "/foo")
infos, err := fs.ReadDir("/foo")
if err != nil {
t.Fatalf("%s: ReadDir: %s", label, err)
}
if len(infos) != 1 || infos[0].Name() != "bar" {
t.Fatalf("%s: ReadDir: got %v, want file 'bar'", label, infos)
}
r, err = fs.Open(path)
if err != nil {
t.Fatalf("%s: Open: %s", label, err)
}
output, err = ioutil.ReadAll(r)
if err != nil {
t.Fatalf("%s: ReadAll: %s", label, err)
}
err = r.Close()
if err != nil {
t.Fatalf("%s: r.Close: %s", label, err)
}
if !bytes.Equal(output, input) {
t.Errorf("%s: got output %q, want %q", label, output, input)
}
err = fs.Remove(path)
if err != nil {
t.Errorf("%s: Remove(%q): %s", label, path, err)
}
PathDoesNotExist(t, label, fs, path)
}