// TestFsListDirEmpty tests listing the directories from an empty directory
func TestFsListDirEmpty(t *testing.T) {
skipIfNotOk(t)
objs, dirs, err := fs.NewLister().SetLevel(1).Start(remote, "").GetAll()
require.NoError(t, err)
assert.Equal(t, []string{}, objsToNames(objs))
assert.Equal(t, []string{}, dirsToNames(dirs))
}
func TestDeduplicateRename(t *testing.T) {
if *RemoteName != "TestDrive:" {
t.Skip("Can only test deduplicate on google drive")
}
r := NewRun(t)
defer r.Finalise()
file1 := r.WriteUncheckedObject("one.txt", "This is one", t1)
file2 := r.WriteUncheckedObject("one.txt", "This is one too", t2)
file3 := r.WriteUncheckedObject("one.txt", "This is another one", t3)
r.checkWithDuplicates(t, file1, file2, file3)
err := fs.Deduplicate(r.fremote, fs.DeduplicateRename)
require.NoError(t, err)
list := fs.NewLister().Start(r.fremote, "")
for {
o, err := list.GetObject()
require.NoError(t, err)
// Check if we are finished
if o == nil {
break
}
remote := o.Remote()
if remote != "one-1.txt" &&
remote != "one-2.txt" &&
remote != "one-3.txt" {
t.Errorf("Bad file name after rename %q", remote)
}
size := o.Size()
if size != file1.Size && size != file2.Size && size != file3.Size {
t.Errorf("Size not one of the object sizes %d", size)
}
}
}
// NewRun initialise the remote and local for testing and returns a
// run object. Call this from the tests.
//
// r.flocal is an empty local Fs
// r.fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func NewRun(t *testing.T) *Run {
var r *Run
if *Individual {
r = newRun()
} else {
// If not individual, use the global one with the clean method overridden
r = new(Run)
*r = *oneRun
r.cleanRemote = func() {
list := fs.NewLister().Start(r.fremote, "")
for {
o, err := list.GetObject()
if err != nil {
t.Fatalf("Error listing: %v", err)
}
// Check if we are finished
if o == nil {
break
}
err = o.Remove()
if err != nil {
t.Errorf("Error removing file: %v", err)
}
}
// Check remote is empty
fstest.CheckItems(t, r.fremote)
}
}
r.Logf = t.Logf
r.Fatalf = t.Fatalf
r.Logf("Remote %q, Local %q, Modify Window %q", r.fremote, r.flocal, fs.Config.ModifyWindow)
return r
}
// TestFsListDirRoot tests that DirList works in the root
func TestFsListDirRoot(t *testing.T) {
skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName)
require.NoError(t, err)
dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote, "").GetDirs()
require.NoError(t, err)
assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found")
}
// TestFsListLevel2 tests List works for 2 levels
func TestFsListLevel2(t *testing.T) {
skipIfNotOk(t)
objs, dirs, err := fs.NewLister().SetLevel(2).Start(remote, "").GetAll()
if err == fs.ErrorLevelNotSupported {
return
}
require.NoError(t, err)
assert.Equal(t, []string{file1.Path}, objsToNames(objs))
assert.Equal(t, []string{`hello_ sausage`, `hello_ sausage/êé`}, dirsToNames(dirs))
}
// TestFsListDirRoot tests that DirList works in the root
func TestFsListDirRoot(t *testing.T) {
skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName)
if err != nil {
t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
}
dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote, "").GetDirs()
require.NoError(t, err)
assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found")
}
// read the directory
func (d *Dir) readDir() error {
d.mu.Lock()
defer d.mu.Unlock()
when := time.Now()
if d.read.IsZero() {
fs.Debug(d.path, "Reading directory")
} else {
age := when.Sub(d.read)
if age < dirCacheTime {
return nil
}
fs.Debug(d.path, "Re-reading directory (%v old)", age)
}
objs, dirs, err := fs.NewLister().SetLevel(1).Start(d.f, d.path).GetAll()
if err == fs.ErrorDirNotFound {
// We treat directory not found as empty because we
// create directories on the fly
} else if err != nil {
return err
}
// NB when we re-read a directory after its cache has expired
// we drop the old files which should lead to correct
// behaviour but may not be very efficient.
// Keep a note of the previous contents of the directory
oldItems := d.items
// Cache the items by name
d.items = make(map[string]*DirEntry, len(objs)+len(dirs))
for _, obj := range objs {
name := path.Base(obj.Remote())
d.items[name] = &DirEntry{
o: obj,
node: nil,
}
}
for _, dir := range dirs {
name := path.Base(dir.Remote())
// Use old dir value if it exists
if oldItem, ok := oldItems[name]; ok {
if _, ok := oldItem.o.(*fs.Dir); ok {
d.items[name] = oldItem
continue
}
}
d.items[name] = &DirEntry{
o: dir,
node: nil,
}
}
d.read = when
return nil
}
// CheckListingWithPrecision checks the fs to see if it has the
// expected contents with the given precision.
//
// If expectedDirs is non nil then we check those too. Note that no
// directories returned is also OK as some remotes don't return
// directories.
func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs []string, precision time.Duration) {
is := NewItems(items)
oldErrors := fs.Stats.GetErrors()
var objs []fs.Object
var dirs []*fs.Dir
var err error
var retries = *listRetries
sleep := time.Second / 2
for i := 1; i <= retries; i++ {
objs, dirs, err = fs.NewLister().Start(f, "").GetAll()
if err != nil && err != fs.ErrorDirNotFound {
t.Fatalf("Error listing: %v", err)
}
if len(objs) == len(items) && (expectedDirs == nil || len(dirs) == 0 || len(dirs) == len(expectedDirs)) {
// Put an extra sleep in if we did any retries just to make sure it really
// is consistent (here is looking at you Amazon Drive!)
if i != 1 {
extraSleep := 5*time.Second + sleep
t.Logf("Sleeping for %v just to make sure", extraSleep)
time.Sleep(extraSleep)
}
break
}
sleep *= 2
t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
time.Sleep(sleep)
if doDirCacheFlush := f.Features().DirCacheFlush; doDirCacheFlush != nil {
t.Logf("Flushing the directory cache")
doDirCacheFlush()
}
}
for _, obj := range objs {
require.NotNil(t, obj)
is.Find(t, obj, precision)
}
is.Done(t)
// Don't notice an error when listing an empty directory
if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 {
fs.Stats.ResetErrors()
}
// Check the directories - ignore if no directories returned
// for remotes which can't do directories
if expectedDirs != nil && len(dirs) != 0 {
actualDirs := []string{}
for _, dir := range dirs {
actualDirs = append(actualDirs, dir.Name)
}
sort.Strings(actualDirs)
sort.Strings(expectedDirs)
assert.Equal(t, expectedDirs, actualDirs, "directories")
}
}
// TestFsListSubdir tests List works for a subdirectory
func TestFsListSubdir(t *testing.T) {
skipIfNotOk(t)
fileName := file2.Path
if runtime.GOOS == "windows" {
fileName = file2.WinPath
}
dir, _ := path.Split(fileName)
dir = dir[:len(dir)-1]
objs, dirs, err := fs.NewLister().Start(remote, dir).GetAll()
require.NoError(t, err)
require.Len(t, objs, 1)
assert.Equal(t, fileName, objs[0].Remote())
require.Len(t, dirs, 0)
}
// reads the remote tree into dir
func (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) {
objs, dirs, err := fs.NewLister().SetLevel(1).Start(r.fremote, filepath).GetAll()
if err == fs.ErrorDirNotFound {
return
}
require.NoError(t, err)
for _, obj := range objs {
dir[fmt.Sprintf("%s %d", obj.Remote(), obj.Size())] = struct{}{}
}
for _, d := range dirs {
name := d.Remote()
dir[name+"/"] = struct{}{}
r.readRemote(t, dir, name)
}
}
// TestFsListDirFile2 tests the files are correctly uploaded
func TestFsListDirFile2(t *testing.T) {
skipIfNotOk(t)
var objNames, dirNames []string
for i := 1; i <= eventualConsistencyRetries; i++ {
objs, dirs, err := fs.NewLister().SetLevel(1).Start(remote, "").GetAll()
require.NoError(t, err)
objNames = objsToNames(objs)
dirNames = dirsToNames(dirs)
if len(objNames) >= 1 && len(dirNames) >= 1 {
break
}
t.Logf("Sleeping for 1 second for TestFsListDirFile2 eventual consistency: %d/%d", i, eventualConsistencyRetries)
time.Sleep(1 * time.Second)
}
assert.Equal(t, []string{`hello_ sausage`}, dirNames)
assert.Equal(t, []string{file1.Path}, objNames)
}
// TestFsListSubdir tests List works for a subdirectory
func TestFsListSubdir(t *testing.T) {
skipIfNotOk(t)
fileName := file2.Path
var err error
var objs []fs.Object
var dirs []*fs.Dir
for i := 0; i < 2; i++ {
dir, _ := path.Split(fileName)
dir = dir[:len(dir)-1]
objs, dirs, err = fs.NewLister().Start(remote, dir).GetAll()
if err != fs.ErrorDirNotFound {
break
}
fileName = file2.WinPath
}
require.NoError(t, err)
require.Len(t, objs, 1)
assert.Equal(t, fileName, objs[0].Remote())
require.Len(t, dirs, 0)
}
// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
f, err := fs.NewFs(t.remote)
if err != nil {
return err
}
dirs, err := fs.NewLister().SetLevel(1).Start(f, "").GetDirs()
for _, dir := range dirs {
if fstest.MatchTestRemote.MatchString(dir.Name) {
log.Printf("Purging %s%s", t.remote, dir.Name)
dir, err := fs.NewFs(t.remote + dir.Name)
if err != nil {
return err
}
err = fs.Purge(dir)
if err != nil {
return err
}
}
}
return nil
}
// CheckListingWithPrecision checks the fs to see if it has the
// expected contents with the given precision.
func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
is := NewItems(items)
oldErrors := fs.Stats.GetErrors()
var objs []fs.Object
var err error
const retries = 6
sleep := time.Second / 2
for i := 1; i <= retries; i++ {
objs, err = fs.NewLister().Start(f, "").GetObjects()
if err != nil && err != fs.ErrorDirNotFound {
t.Fatalf("Error listing: %v", err)
}
if len(objs) == len(items) {
// Put an extra sleep in if we did any retries just to make sure it really
// is consistent (here is looking at you Amazon Cloud Drive!)
if i != 1 {
extraSleep := 5*time.Second + sleep
t.Logf("Sleeping for %v just to make sure", extraSleep)
time.Sleep(extraSleep)
}
break
}
sleep *= 2
t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
time.Sleep(sleep)
}
for _, obj := range objs {
require.NotNil(t, obj)
is.Find(t, obj, precision)
}
is.Done(t)
// Don't notice an error when listing an empty directory
if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 {
fs.Stats.ResetErrors()
}
}