// ForceMkdir creates the remote
func (r *Run) ForceMkdir(f fs.Fs) {
err := f.Mkdir("")
if err != nil {
r.Fatalf("Failed to mkdir %q: %v", f, err)
}
r.mkdir[f.String()] = true
}
// mount the file system
//
// The mount point will be ready when this returns.
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(f fs.Fs, mountpoint string) (<-chan error, error) {
fs.Debug(f, "Mounting on %q", mountpoint)
c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...)
if err != nil {
return nil, err
}
filesys := &FS{
f: f,
}
// Serve the mount point in the background returning error to errChan
errChan := make(chan error, 1)
go func() {
err := fusefs.Serve(c, filesys)
closeErr := c.Close()
if err == nil {
err = closeErr
}
errChan <- err
}()
// check if the mount process has an error to report
<-c.Ready
if err := c.MountError; err != nil {
return nil, err
}
return errChan, nil
}
// CheckListingWithPrecision checks the fs to see if it has the
// expected contents with the given precision.
func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
is := NewItems(items)
oldErrors := fs.Stats.GetErrors()
var objs []fs.Object
for i := 1; i <= 5; i++ {
objs = nil
for obj := range f.List() {
objs = append(objs, obj)
}
if len(objs) == len(items) {
break
}
t.Logf("Sleeping for 1 second for list eventual consistency: %d/5", i)
time.Sleep(1 * time.Second)
}
for _, obj := range objs {
if obj == nil {
t.Errorf("Unexpected nil in List()")
continue
}
is.Find(t, obj, precision)
}
is.Done(t)
// Don't notice an error when listing an empty directory
if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 {
fs.Stats.ResetErrors()
}
}
// WriteObjectTo writes an object to the fs, remote passed in
func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) fstest.Item {
put := f.Put
if useUnchecked {
put = f.Features().PutUnchecked
if put == nil {
r.Fatalf("Fs doesn't support PutUnchecked")
}
}
r.Mkdir(f)
const maxTries = 10
for tries := 1; ; tries++ {
in := bytes.NewBufferString(content)
objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil)
_, err := put(in, objinfo)
if err == nil {
break
}
// Retry if err returned a retry error
if fs.IsRetryError(err) && tries < maxTries {
r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err)
time.Sleep(2 * time.Second)
continue
}
r.Fatalf("Failed to put %q to %q: %v", remote, f, err)
}
return fstest.NewItem(remote, content, modTime)
}
// Checks the fs to see if it has the expected contents
func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
is := NewItems(items)
for obj := range f.List() {
if obj == nil {
t.Errorf("Unexpected nil in List()")
continue
}
is.Find(t, obj, precision)
}
is.Done(t)
}
// CheckListingWithPrecision checks the fs to see if it has the
// expected contents with the given precision.
//
// If expectedDirs is non nil then we check those too. Note that no
// directories returned is also OK as some remotes don't return
// directories.
func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs []string, precision time.Duration) {
is := NewItems(items)
oldErrors := fs.Stats.GetErrors()
var objs []fs.Object
var dirs []*fs.Dir
var err error
var retries = *listRetries
sleep := time.Second / 2
for i := 1; i <= retries; i++ {
objs, dirs, err = fs.NewLister().Start(f, "").GetAll()
if err != nil && err != fs.ErrorDirNotFound {
t.Fatalf("Error listing: %v", err)
}
if len(objs) == len(items) && (expectedDirs == nil || len(dirs) == 0 || len(dirs) == len(expectedDirs)) {
// Put an extra sleep in if we did any retries just to make sure it really
// is consistent (here is looking at you Amazon Drive!)
if i != 1 {
extraSleep := 5*time.Second + sleep
t.Logf("Sleeping for %v just to make sure", extraSleep)
time.Sleep(extraSleep)
}
break
}
sleep *= 2
t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
time.Sleep(sleep)
if doDirCacheFlush := f.Features().DirCacheFlush; doDirCacheFlush != nil {
t.Logf("Flushing the directory cache")
doDirCacheFlush()
}
}
for _, obj := range objs {
require.NotNil(t, obj)
is.Find(t, obj, precision)
}
is.Done(t)
// Don't notice an error when listing an empty directory
if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 {
fs.Stats.ResetErrors()
}
// Check the directories - ignore if no directories returned
// for remotes which can't do directories
if expectedDirs != nil && len(dirs) != 0 {
actualDirs := []string{}
for _, dir := range dirs {
actualDirs = append(actualDirs, dir.Name)
}
sort.Strings(actualDirs)
sort.Strings(expectedDirs)
assert.Equal(t, expectedDirs, actualDirs, "directories")
}
}
// Checks the fs to see if it has the expected contents
func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
is := NewItems(items)
oldErrors := fs.Stats.GetErrors()
for obj := range f.List() {
if obj == nil {
t.Errorf("Unexpected nil in List()")
continue
}
is.Find(t, obj, precision)
}
is.Done(t)
// Don't notice an error when listing an empty directory
if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 {
fs.Stats.ResetErrors()
}
}
// WriteObjectTo writes an object to the fs, remote passed in
func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) fstest.Item {
put := f.Put
if useUnchecked {
if fPutUnchecked, ok := f.(fs.PutUncheckeder); ok {
put = fPutUnchecked.PutUnchecked
} else {
r.Fatalf("Fs doesn't support PutUnchecked")
}
}
const maxTries = 5
if !r.mkdir[f.String()] {
err := f.Mkdir()
if err != nil {
r.Fatalf("Failed to mkdir %q: %v", f, err)
}
r.mkdir[f.String()] = true
}
for tries := 1; ; tries++ {
in := bytes.NewBufferString(content)
objinfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil)
_, err := put(in, objinfo)
if err == nil {
break
}
// Retry if err returned a retry error
if fs.IsRetryError(err) && tries < maxTries {
r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err)
continue
}
r.Fatalf("Failed to put %q to %q: %v", remote, f, err)
}
return fstest.NewItem(remote, content, modTime)
}
// CheckListingWithPrecision checks the fs to see if it has the
// expected contents with the given precision.
func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
is := NewItems(items)
oldErrors := fs.Stats.GetErrors()
var objs []fs.Object
const retries = 6
sleep := time.Second / 2
for i := 1; i <= retries; i++ {
objs = nil
for obj := range f.List() {
objs = append(objs, obj)
}
if len(objs) == len(items) {
// Put an extra sleep in if we did any retries just to make sure it really
// is consistent (here is looking at you Amazon Cloud Drive!)
if i != 1 {
extraSleep := 5*time.Second + sleep
t.Logf("Sleeping for %v just to make sure", extraSleep)
time.Sleep(extraSleep)
}
break
}
sleep *= 2
t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
time.Sleep(sleep)
}
for _, obj := range objs {
if obj == nil {
t.Errorf("Unexpected nil in List()")
continue
}
is.Find(t, obj, precision)
}
is.Done(t)
// Don't notice an error when listing an empty directory
if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 {
fs.Stats.ResetErrors()
}
}
func skipIfCantDedupe(t *testing.T, f fs.Fs) {
if f.Features().PutUnchecked == nil {
t.Skip("Can't test deduplicate - no PutUnchecked")
}
if !f.Features().DuplicateFiles {
t.Skip("Can't test deduplicate - no duplicate files possible")
}
if !f.Hashes().Contains(fs.HashMD5) {
t.Skip("Can't test deduplicate - MD5 not supported")
}
}
// WriteObjectTo writes an object to the fs, remote passed in
func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time) fstest.Item {
const maxTries = 5
if !r.mkdir[f.String()] {
err := f.Mkdir()
if err != nil {
r.Fatalf("Failed to mkdir %q: %v", f, err)
}
r.mkdir[f.String()] = true
}
for tries := 1; ; tries++ {
in := bytes.NewBufferString(content)
_, err := f.Put(in, remote, modTime, int64(len(content)))
if err == nil {
break
}
// Retry if err returned a retry error
if retry, ok := err.(fs.Retry); ok && retry.Retry() && tries < maxTries {
r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err)
continue
}
r.Fatalf("Failed to put %q to %q: %v", remote, f, err)
}
return fstest.NewItem(remote, content, modTime)
}
// CheckListing checks the fs to see if it has the expected contents
func CheckListing(t *testing.T, f fs.Fs, items []Item) {
precision := f.Precision()
CheckListingWithPrecision(t, f, items, precision)
}
// Mkdir creates the remote if it hasn't been created already
func (r *Run) Mkdir(f fs.Fs) {
if !r.mkdir[f.String()] {
r.ForceMkdir(f)
}
}