// ForceMkdir creates the remote func (r *Run) ForceMkdir(f fs.Fs) { err := f.Mkdir("") if err != nil { r.Fatalf("Failed to mkdir %q: %v", f, err) } r.mkdir[f.String()] = true }
// mount the file system // // The mount point will be ready when this returns. // // returns an error, and an error channel for the serve process to // report an error when fusermount is called. func mount(f fs.Fs, mountpoint string) (<-chan error, error) { fs.Debug(f, "Mounting on %q", mountpoint) c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...) if err != nil { return nil, err } filesys := &FS{ f: f, } // Serve the mount point in the background returning error to errChan errChan := make(chan error, 1) go func() { err := fusefs.Serve(c, filesys) closeErr := c.Close() if err == nil { err = closeErr } errChan <- err }() // check if the mount process has an error to report <-c.Ready if err := c.MountError; err != nil { return nil, err } return errChan, nil }
// CheckListingWithPrecision checks the fs to see if it has the // expected contents with the given precision. func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) { is := NewItems(items) oldErrors := fs.Stats.GetErrors() var objs []fs.Object for i := 1; i <= 5; i++ { objs = nil for obj := range f.List() { objs = append(objs, obj) } if len(objs) == len(items) { break } t.Logf("Sleeping for 1 second for list eventual consistency: %d/5", i) time.Sleep(1 * time.Second) } for _, obj := range objs { if obj == nil { t.Errorf("Unexpected nil in List()") continue } is.Find(t, obj, precision) } is.Done(t) // Don't notice an error when listing an empty directory if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 { fs.Stats.ResetErrors() } }
// WriteObjectTo writes an object to the fs, remote passed in func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) fstest.Item { put := f.Put if useUnchecked { put = f.Features().PutUnchecked if put == nil { r.Fatalf("Fs doesn't support PutUnchecked") } } r.Mkdir(f) const maxTries = 10 for tries := 1; ; tries++ { in := bytes.NewBufferString(content) objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil) _, err := put(in, objinfo) if err == nil { break } // Retry if err returned a retry error if fs.IsRetryError(err) && tries < maxTries { r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err) time.Sleep(2 * time.Second) continue } r.Fatalf("Failed to put %q to %q: %v", remote, f, err) } return fstest.NewItem(remote, content, modTime) }
// Checks the fs to see if it has the expected contents func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) { is := NewItems(items) for obj := range f.List() { if obj == nil { t.Errorf("Unexpected nil in List()") continue } is.Find(t, obj, precision) } is.Done(t) }
// CheckListingWithPrecision checks the fs to see if it has the // expected contents with the given precision. // // If expectedDirs is non nil then we check those too. Note that no // directories returned is also OK as some remotes don't return // directories. func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs []string, precision time.Duration) { is := NewItems(items) oldErrors := fs.Stats.GetErrors() var objs []fs.Object var dirs []*fs.Dir var err error var retries = *listRetries sleep := time.Second / 2 for i := 1; i <= retries; i++ { objs, dirs, err = fs.NewLister().Start(f, "").GetAll() if err != nil && err != fs.ErrorDirNotFound { t.Fatalf("Error listing: %v", err) } if len(objs) == len(items) && (expectedDirs == nil || len(dirs) == 0 || len(dirs) == len(expectedDirs)) { // Put an extra sleep in if we did any retries just to make sure it really // is consistent (here is looking at you Amazon Drive!) if i != 1 { extraSleep := 5*time.Second + sleep t.Logf("Sleeping for %v just to make sure", extraSleep) time.Sleep(extraSleep) } break } sleep *= 2 t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries) time.Sleep(sleep) if doDirCacheFlush := f.Features().DirCacheFlush; doDirCacheFlush != nil { t.Logf("Flushing the directory cache") doDirCacheFlush() } } for _, obj := range objs { require.NotNil(t, obj) is.Find(t, obj, precision) } is.Done(t) // Don't notice an error when listing an empty directory if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 { fs.Stats.ResetErrors() } // Check the directories - ignore if no directories returned // for remotes which can't do directories if expectedDirs != nil && len(dirs) != 0 { actualDirs := []string{} for _, dir := range dirs { actualDirs = append(actualDirs, dir.Name) } sort.Strings(actualDirs) sort.Strings(expectedDirs) assert.Equal(t, expectedDirs, actualDirs, "directories") } }
// Checks the fs to see if it has the expected contents func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) { is := NewItems(items) oldErrors := fs.Stats.GetErrors() for obj := range f.List() { if obj == nil { t.Errorf("Unexpected nil in List()") continue } is.Find(t, obj, precision) } is.Done(t) // Don't notice an error when listing an empty directory if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 { fs.Stats.ResetErrors() } }
// WriteObjectTo writes an object to the fs, remote passed in func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) fstest.Item { put := f.Put if useUnchecked { if fPutUnchecked, ok := f.(fs.PutUncheckeder); ok { put = fPutUnchecked.PutUnchecked } else { r.Fatalf("Fs doesn't support PutUnchecked") } } const maxTries = 5 if !r.mkdir[f.String()] { err := f.Mkdir() if err != nil { r.Fatalf("Failed to mkdir %q: %v", f, err) } r.mkdir[f.String()] = true } for tries := 1; ; tries++ { in := bytes.NewBufferString(content) objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil) _, err := put(in, objinfo) if err == nil { break } // Retry if err returned a retry error if fs.IsRetryError(err) && tries < maxTries { r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err) continue } r.Fatalf("Failed to put %q to %q: %v", remote, f, err) } return fstest.NewItem(remote, content, modTime) }
// CheckListingWithPrecision checks the fs to see if it has the // expected contents with the given precision. func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) { is := NewItems(items) oldErrors := fs.Stats.GetErrors() var objs []fs.Object const retries = 6 sleep := time.Second / 2 for i := 1; i <= retries; i++ { objs = nil for obj := range f.List() { objs = append(objs, obj) } if len(objs) == len(items) { // Put an extra sleep in if we did any retries just to make sure it really // is consistent (here is looking at you Amazon Cloud Drive!) if i != 1 { extraSleep := 5*time.Second + sleep t.Logf("Sleeping for %v just to make sure", extraSleep) time.Sleep(extraSleep) } break } sleep *= 2 t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries) time.Sleep(sleep) } for _, obj := range objs { if obj == nil { t.Errorf("Unexpected nil in List()") continue } is.Find(t, obj, precision) } is.Done(t) // Don't notice an error when listing an empty directory if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 { fs.Stats.ResetErrors() } }
func skipIfCantDedupe(t *testing.T, f fs.Fs) { if f.Features().PutUnchecked == nil { t.Skip("Can't test deduplicate - no PutUnchecked") } if !f.Features().DuplicateFiles { t.Skip("Can't test deduplicate - no duplicate files possible") } if !f.Hashes().Contains(fs.HashMD5) { t.Skip("Can't test deduplicate - MD5 not supported") } }
// WriteObjectTo writes an object to the fs, remote passed in func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time) fstest.Item { const maxTries = 5 if !r.mkdir[f.String()] { err := f.Mkdir() if err != nil { r.Fatalf("Failed to mkdir %q: %v", f, err) } r.mkdir[f.String()] = true } for tries := 1; ; tries++ { in := bytes.NewBufferString(content) _, err := f.Put(in, remote, modTime, int64(len(content))) if err == nil { break } // Retry if err returned a retry error if retry, ok := err.(fs.Retry); ok && retry.Retry() && tries < maxTries { r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err) continue } r.Fatalf("Failed to put %q to %q: %v", remote, f, err) } return fstest.NewItem(remote, content, modTime) }
// CheckListing checks the fs to see if it has the expected contents func CheckListing(t *testing.T, f fs.Fs, items []Item) { precision := f.Precision() CheckListingWithPrecision(t, f, items, precision) }
// Mkdir creates the remote if it hasn't been created already func (r *Run) Mkdir(f fs.Fs) { if !r.mkdir[f.String()] { r.ForceMkdir(f) } }