// listFiles walks the path returning files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
// List the objects
err := f.list(dir, out.Level(), "", 0, false, func(remote string, object *api.File, isDirectory bool) error {
if isDirectory {
dir := &fs.Dir{
Name: remote,
Bytes: -1,
Count: -1,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
} else {
if o := f.newFsObjectWithInfo(remote, object); o != nil {
if out.Add(o) {
return fs.ErrorListAborted
}
}
}
return nil
})
if err != nil {
out.SetError(err)
}
}
// List walks the path returning a channel of Objects
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
level := out.Level()
switch level {
case 1:
f.listOneLevel(out, dir)
case fs.MaxLevel:
f.list(out, dir)
default:
out.SetError(fs.ErrorLevelNotSupported)
}
}
// List walks the path returning a channel of Objects
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
listItem := func(remote string, object *yandex.ResourceInfoResponse, isDirectory bool) error {
if isDirectory {
t, err := time.Parse(time.RFC3339Nano, object.Modified)
if err != nil {
return err
}
dir := &fs.Dir{
Name: remote,
When: t,
Bytes: int64(object.Size),
Count: -1,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
} else {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
}
var err error
switch out.Level() {
case 1:
if dir == "" {
err = f.listDir(listItem)
} else {
err = f.list(dir, listItem)
}
case fs.MaxLevel:
err = f.list(dir, listItem)
default:
out.SetError(fs.ErrorLevelNotSupported)
}
if err != nil {
// FIXME
// if err == swift.ContainerNotFound {
// err = fs.ErrorDirNotFound
// }
out.SetError(err)
}
}
// List the path into out
//
// Ignores everything which isn't Storable, eg links etc
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
dir = filterFragment(f.cleanUtf8(dir))
root := filepath.Join(f.root, dir)
_, err := os.Stat(root)
if err != nil {
out.SetError(fs.ErrorDirNotFound)
return
}
in := make(chan listArgs, out.Buffer())
var wg sync.WaitGroup // sync closing of go routines
var traversing sync.WaitGroup // running directory traversals
// Start the process
traversing.Add(1)
in <- listArgs{remote: dir, dirpath: root, level: out.Level() - 1}
for i := 0; i < fs.Config.Checkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for job := range in {
if out.IsFinished() {
continue
}
newJobs := f.list(out, job.remote, job.dirpath, job.level)
// Now we have traversed this directory, send
// these ones off for traversal
if len(newJobs) != 0 {
traversing.Add(len(newJobs))
go func() {
for _, newJob := range newJobs {
in <- newJob
}
}()
}
traversing.Done()
}
}()
}
// Wait for traversal to finish
traversing.Wait()
close(in)
wg.Wait()
}
// listFiles lists files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.bucket == "" {
// Return no objects at top level list
out.SetError(errors.New("can't list objects at root - choose a bucket using lsd"))
return
}
// List the objects and directories
err := f.list(dir, out.Level(), func(remote string, object *s3.Object, isDirectory bool) error {
if isDirectory {
size := int64(0)
if object.Size != nil {
size = *object.Size
}
dir := &fs.Dir{
Name: remote,
Bytes: size,
Count: 0,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
} else {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
})
if err != nil {
if awsErr, ok := err.(awserr.RequestFailure); ok {
if awsErr.StatusCode() == http.StatusNotFound {
err = fs.ErrorDirNotFound
}
}
out.SetError(err)
}
}
// listFiles walks the path returning files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
// List the objects
last := ""
err := f.list(dir, out.Level(), "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
if isDirectory {
dir := &fs.Dir{
Name: remote,
Bytes: -1,
Count: -1,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
} else {
if remote == last {
remote = object.UploadTimestamp.AddVersion(remote)
} else {
last = remote
}
// hide objects represent deleted files which we don't list
if object.Action == "hide" {
return nil
}
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
})
if err != nil {
out.SetError(err)
}
}
// listFiles walks the path returning a channel of Objects
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.container == "" {
out.SetError(errors.New("can't list objects at root - choose a container using lsd"))
return
}
// List the objects
err := f.list(dir, out.Level(), func(remote string, object *swift.Object, isDirectory bool) error {
if isDirectory {
dir := &fs.Dir{
Name: remote,
Bytes: object.Bytes,
Count: 0,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
} else {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
// Storable does a full metadata read on 0 size objects which might be dynamic large objects
if o.Storable() {
if out.Add(o) {
return fs.ErrorListAborted
}
}
}
return nil
})
if err != nil {
if err == swift.ContainerNotFound {
err = fs.ErrorDirNotFound
}
out.SetError(err)
}
}
// listFiles lists files and directories to out
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.bucket == "" {
out.SetError(errors.New("can't list objects at root - choose a bucket using lsd"))
return
}
// List the objects
err := f.list(dir, out.Level(), func(remote string, object *storage.Object, isDirectory bool) error {
if isDirectory {
dir := &fs.Dir{
Name: remote,
Bytes: int64(object.Size),
Count: 0,
}
if out.AddDir(dir) {
return fs.ErrorListAborted
}
} else {
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
})
if err != nil {
if gErr, ok := err.(*googleapi.Error); ok {
if gErr.Code == http.StatusNotFound {
err = fs.ErrorDirNotFound
}
}
out.SetError(err)
}
}
// listDir lists the directory using a recursive list from the root
//
// It does this in parallel, calling f.ListDir to do the actual reading
func listDir(f ListDirer, out fs.ListOpts, dirID string, path string) {
// Start some directory listing go routines
var wg sync.WaitGroup // sync closing of go routines
var traversing sync.WaitGroup // running directory traversals
buffer := out.Buffer()
in := make(chan ListDirJob, buffer)
for i := 0; i < buffer; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for job := range in {
jobs, err := f.ListDir(out, job)
if err != nil {
out.SetError(err)
fs.Debug(f, "Error reading %s: %s", path, err)
} else {
traversing.Add(len(jobs))
go func() {
// Now we have traversed this directory, send these
// jobs off for traversal in the background
for _, job := range jobs {
in <- job
}
}()
}
traversing.Done()
}
}()
}
// Start the process
traversing.Add(1)
in <- ListDirJob{DirID: dirID, Path: path, Depth: out.Level() - 1}
traversing.Wait()
close(in)
wg.Wait()
}
