Example #1
0
// listBuckets lists the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
	defer out.Finished()
	if dir != "" {
		out.SetError(fs.ErrorListOnlyRoot)
		return
	}
	if f.projectNumber == "" {
		out.SetError(errors.New("can't list buckets without project number"))
		return
	}
	listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks)
	for {
		buckets, err := listBuckets.Do()
		if err != nil {
			out.SetError(err)
			return
		}
		for _, bucket := range buckets.Items {
			dir := &fs.Dir{
				Name:  bucket.Name,
				Bytes: 0,
				Count: 0,
			}
			if out.AddDir(dir) {
				return
			}
		}
		if buckets.NextPageToken == "" {
			break
		}
		listBuckets.PageToken(buckets.NextPageToken)
	}
}
Example #2
0
// listFiles walks the path returning files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
	defer out.Finished()
	// List the objects
	err := f.list(dir, out.Level(), "", 0, false, func(remote string, object *api.File, isDirectory bool) error {
		if isDirectory {
			dir := &fs.Dir{
				Name:  remote,
				Bytes: -1,
				Count: -1,
			}
			if out.AddDir(dir) {
				return fs.ErrorListAborted
			}
		} else {
			if o := f.newFsObjectWithInfo(remote, object); o != nil {
				if out.Add(o) {
					return fs.ErrorListAborted
				}
			}
		}
		return nil
	})
	if err != nil {
		out.SetError(err)
	}
}
Example #3
0
// List walks the path returning a channel of Objects
func (f *Fs) List(out fs.ListOpts, dir string) {
	defer out.Finished()
	level := out.Level()
	switch level {
	case 1:
		f.listOneLevel(out, dir)
	case fs.MaxLevel:
		f.list(out, dir)
	default:
		out.SetError(fs.ErrorLevelNotSupported)
	}
}
Example #4
0
// List walks the path returning a channel of Objects
func (f *Fs) List(out fs.ListOpts, dir string) {
	defer out.Finished()

	listItem := func(remote string, object *yandex.ResourceInfoResponse, isDirectory bool) error {
		if isDirectory {
			t, err := time.Parse(time.RFC3339Nano, object.Modified)
			if err != nil {
				return err
			}
			dir := &fs.Dir{
				Name:  remote,
				When:  t,
				Bytes: int64(object.Size),
				Count: -1,
			}
			if out.AddDir(dir) {
				return fs.ErrorListAborted
			}
		} else {
			o, err := f.newObjectWithInfo(remote, object)
			if err != nil {
				return err
			}
			if out.Add(o) {
				return fs.ErrorListAborted
			}
		}
		return nil
	}

	var err error
	switch out.Level() {
	case 1:
		if dir == "" {
			err = f.listDir(listItem)
		} else {
			err = f.list(dir, listItem)
		}
	case fs.MaxLevel:
		err = f.list(dir, listItem)
	default:
		out.SetError(fs.ErrorLevelNotSupported)
	}

	if err != nil {
		// FIXME
		// if err == swift.ContainerNotFound {
		// 	err = fs.ErrorDirNotFound
		// }
		out.SetError(err)
	}
}
Example #5
0
// List the path into out
//
// Ignores everything which isn't Storable, eg links etc
func (f *Fs) List(out fs.ListOpts, dir string) {
	defer out.Finished()
	dir = filterFragment(f.cleanUtf8(dir))
	root := filepath.Join(f.root, dir)
	_, err := os.Stat(root)
	if err != nil {
		out.SetError(fs.ErrorDirNotFound)
		return
	}

	in := make(chan listArgs, out.Buffer())
	var wg sync.WaitGroup         // sync closing of go routines
	var traversing sync.WaitGroup // running directory traversals

	// Start the process
	traversing.Add(1)
	in <- listArgs{remote: dir, dirpath: root, level: out.Level() - 1}
	for i := 0; i < fs.Config.Checkers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for job := range in {
				if out.IsFinished() {
					continue
				}
				newJobs := f.list(out, job.remote, job.dirpath, job.level)
				// Now we have traversed this directory, send
				// these ones off for traversal
				if len(newJobs) != 0 {
					traversing.Add(len(newJobs))
					go func() {
						for _, newJob := range newJobs {
							in <- newJob
						}
					}()
				}
				traversing.Done()
			}
		}()
	}

	// Wait for traversal to finish
	traversing.Wait()
	close(in)
	wg.Wait()
}
Example #6
0
// List walks the path returning iles and directories into out
func (dc *DirCache) List(f ListDirer, out fs.ListOpts, dir string) {
	defer out.Finished()
	err := dc.FindRoot(false)
	if err != nil {
		out.SetError(err)
		return
	}
	id, err := dc.FindDir(dir, false)
	if err != nil {
		out.SetError(err)
		return
	}
	if dir != "" {
		dir += "/"
	}
	listDir(f, out, id, dir)
}
Example #7
0
File: s3.go Project: yut148/rclone
// listFiles lists files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
	defer out.Finished()
	if f.bucket == "" {
		// Return no objects at top level list
		out.SetError(errors.New("can't list objects at root - choose a bucket using lsd"))
		return
	}
	// List the objects and directories
	err := f.list(dir, out.Level(), func(remote string, object *s3.Object, isDirectory bool) error {
		if isDirectory {
			size := int64(0)
			if object.Size != nil {
				size = *object.Size
			}
			dir := &fs.Dir{
				Name:  remote,
				Bytes: size,
				Count: 0,
			}
			if out.AddDir(dir) {
				return fs.ErrorListAborted
			}
		} else {
			o, err := f.newObjectWithInfo(remote, object)
			if err != nil {
				return err
			}
			if out.Add(o) {
				return fs.ErrorListAborted
			}
		}
		return nil
	})
	if err != nil {
		if awsErr, ok := err.(awserr.RequestFailure); ok {
			if awsErr.StatusCode() == http.StatusNotFound {
				err = fs.ErrorDirNotFound
			}
		}
		out.SetError(err)
	}
}
Example #8
0
File: b2.go Project: ncw/rclone
// listBuckets returns all the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
	defer out.Finished()
	if dir != "" {
		out.SetError(fs.ErrorListOnlyRoot)
		return
	}
	err := f.listBucketsToFn(func(bucket *api.Bucket) error {
		dir := &fs.Dir{
			Name:  bucket.Name,
			Bytes: -1,
			Count: -1,
		}
		if out.AddDir(dir) {
			return fs.ErrorListAborted
		}
		return nil
	})
	if err != nil {
		out.SetError(err)
	}
}
Example #9
0
// listContainers lists the containers
func (f *Fs) listContainers(out fs.ListOpts, dir string) {
	defer out.Finished()
	if dir != "" {
		out.SetError(fs.ErrorListOnlyRoot)
		return
	}
	containers, err := f.c.ContainersAll(nil)
	if err != nil {
		out.SetError(err)
		return
	}
	for _, container := range containers {
		dir := &fs.Dir{
			Name:  container.Name,
			Bytes: container.Bytes,
			Count: container.Count,
		}
		if out.AddDir(dir) {
			break
		}
	}
}
Example #10
0
File: b2.go Project: ncw/rclone
// listFiles walks the path returning files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
	defer out.Finished()
	// List the objects
	last := ""
	err := f.list(dir, out.Level(), "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
		if isDirectory {
			dir := &fs.Dir{
				Name:  remote,
				Bytes: -1,
				Count: -1,
			}
			if out.AddDir(dir) {
				return fs.ErrorListAborted
			}
		} else {
			if remote == last {
				remote = object.UploadTimestamp.AddVersion(remote)
			} else {
				last = remote
			}
			// hide objects represent deleted files which we don't list
			if object.Action == "hide" {
				return nil
			}
			o, err := f.newObjectWithInfo(remote, object)
			if err != nil {
				return err
			}
			if out.Add(o) {
				return fs.ErrorListAborted
			}
		}
		return nil
	})
	if err != nil {
		out.SetError(err)
	}
}
Example #11
0
// listFiles walks the path returning a channel of Objects
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
	defer out.Finished()
	if f.container == "" {
		out.SetError(errors.New("can't list objects at root - choose a container using lsd"))
		return
	}
	// List the objects
	err := f.list(dir, out.Level(), func(remote string, object *swift.Object, isDirectory bool) error {
		if isDirectory {
			dir := &fs.Dir{
				Name:  remote,
				Bytes: object.Bytes,
				Count: 0,
			}
			if out.AddDir(dir) {
				return fs.ErrorListAborted
			}
		} else {
			o, err := f.newObjectWithInfo(remote, object)
			if err != nil {
				return err
			}
			// Storable does a full metadata read on 0 size objects which might be dynamic large objects
			if o.Storable() {
				if out.Add(o) {
					return fs.ErrorListAborted
				}
			}
		}
		return nil
	})
	if err != nil {
		if err == swift.ContainerNotFound {
			err = fs.ErrorDirNotFound
		}
		out.SetError(err)
	}
}
Example #12
0
// listFiles lists files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
	defer out.Finished()
	if f.bucket == "" {
		out.SetError(errors.New("can't list objects at root - choose a bucket using lsd"))
		return
	}
	// List the objects
	err := f.list(dir, out.Level(), func(remote string, object *storage.Object, isDirectory bool) error {
		if isDirectory {
			dir := &fs.Dir{
				Name:  remote,
				Bytes: int64(object.Size),
				Count: 0,
			}
			if out.AddDir(dir) {
				return fs.ErrorListAborted
			}
		} else {
			o, err := f.newObjectWithInfo(remote, object)
			if err != nil {
				return err
			}
			if out.Add(o) {
				return fs.ErrorListAborted
			}
		}
		return nil
	})
	if err != nil {
		if gErr, ok := err.(*googleapi.Error); ok {
			if gErr.Code == http.StatusNotFound {
				err = fs.ErrorDirNotFound
			}
		}
		out.SetError(err)
	}
}
Example #13
0
File: s3.go Project: yut148/rclone
// listBuckets lists the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
	defer out.Finished()
	if dir != "" {
		out.SetError(fs.ErrorListOnlyRoot)
		return
	}
	req := s3.ListBucketsInput{}
	resp, err := f.c.ListBuckets(&req)
	if err != nil {
		out.SetError(err)
		return
	}
	for _, bucket := range resp.Buckets {
		dir := &fs.Dir{
			Name:  aws.StringValue(bucket.Name),
			When:  aws.TimeValue(bucket.CreationDate),
			Bytes: -1,
			Count: -1,
		}
		if out.AddDir(dir) {
			break
		}
	}
}