// Upload uploads the chunks from the input
// It retries each chunk maxTries times (with a pause of uploadPause between attempts).
func (rx *resumableUpload) Upload() (*drive.File, error) {
start := int64(0)
buf := make([]byte, chunkSize)
var StatusCode int
for start < rx.ContentLength {
reqSize := rx.ContentLength - start
if reqSize >= int64(chunkSize) {
reqSize = int64(chunkSize)
} else {
buf = buf[:reqSize]
}
// Read the chunk
_, err := io.ReadFull(rx.Media, buf)
if err != nil {
return nil, err
}
// Transfer the chunk
for try := 1; try <= maxTries; try++ {
fs.Debug(rx.remote, "Sending chunk %d length %d, %d/%d", start, reqSize, try, maxTries)
rx.f.beginCall()
StatusCode, err = rx.transferChunk(start, buf)
rx.f.endCall(err)
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
goto success
}
fs.Debug(rx.remote, "Retrying chunk %d/%d, code=%d, err=%v", try, maxTries, StatusCode, err)
}
fs.Debug(rx.remote, "Failed to send chunk")
return nil, fs.RetryErrorf("Chunk upload failed - retry: code=%d, err=%v", StatusCode, err)
success:
start += reqSize
}
// Resume or retry uploads that fail due to connection interruptions or
// any 5xx errors, including:
//
// 500 Internal Server Error
// 502 Bad Gateway
// 503 Service Unavailable
// 504 Gateway Timeout
//
// Use an exponential backoff strategy if any 5xx server error is
// returned when resuming or retrying upload requests. These errors can
// occur if a server is getting overloaded. Exponential backoff can help
// alleviate these kinds of problems during periods of high volume of
// requests or heavy network traffic. Other kinds of requests should not
// be handled by exponential backoff but you can still retry a number of
// them. When retrying these requests, limit the number of times you
// retry them. For example your code could limit to ten retries or less
// before reporting an error.
//
// Handle 404 Not Found errors when doing resumable uploads by starting
// the entire upload over from the beginning.
if rx.ret == nil {
return nil, fs.RetryErrorf("Incomplete upload - retry, last error %d", StatusCode)
}
return rx.ret, nil
}
// Flush is called each time the file or directory is closed.
// Because there can be multiple file descriptors referring to a
// single opened file, Flush can be called multiple times.
func (fh *ReadFileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
fh.mu.Lock()
defer fh.mu.Unlock()
fs.Debug(fh.o, "ReadFileHandle.Flush")
// Ignore the Flush as there is nothing we can sensibly do and
// it seems quite common for Flush to be called from
// different threads each of which have read some data.
if false {
// If Read hasn't been called then ignore the Flush - Release
// will pick it up
if !fh.readCalled {
fs.Debug(fh.o, "ReadFileHandle.Flush ignoring flush on unread handle")
return nil
}
err := fh.close()
if err != nil {
fs.ErrorLog(fh.o, "ReadFileHandle.Flush error: %v", err)
return err
}
}
fs.Debug(fh.o, "ReadFileHandle.Flush OK")
return nil
}
// ReadDirAll reads the contents of the directory
func (d *Dir) ReadDirAll(ctx context.Context) (dirents []fuse.Dirent, err error) {
fs.Debug(d.path, "Dir.ReadDirAll")
err = d.readDir()
if err != nil {
fs.Debug(d.path, "Dir.ReadDirAll error: %v", err)
return nil, err
}
d.mu.RLock()
defer d.mu.RUnlock()
for _, item := range d.items {
var dirent fuse.Dirent
switch x := item.o.(type) {
case fs.Object:
dirent = fuse.Dirent{
// Inode FIXME ???
Type: fuse.DT_File,
Name: path.Base(x.Remote()),
}
case *fs.Dir:
dirent = fuse.Dirent{
// Inode FIXME ???
Type: fuse.DT_Dir,
Name: path.Base(x.Remote()),
}
default:
err = errors.Errorf("unknown type %T", item)
fs.ErrorLog(d.path, "Dir.ReadDirAll error: %v", err)
return nil, err
}
dirents = append(dirents, dirent)
}
fs.Debug(d.path, "Dir.ReadDirAll OK with %d entries", len(dirents))
return dirents, nil
}
// readMetaData gets the info if it hasn't already been fetched
func (o *FsObjectDrive) readMetaData() (err error) {
if o.id != "" {
return nil
}
directory, leaf := splitPath(o.remote)
directoryId, err := o.drive.findDir(directory, false)
if err != nil {
fs.Debug(o, "Couldn't find directory: %s", err)
return fmt.Errorf("Couldn't find directory: %s", err)
}
found, err := o.drive.listAll(directoryId, leaf, false, true, func(item *drive.File) bool {
if item.Title == leaf {
o.setMetaData(item)
return true
}
return false
})
if err != nil {
return err
}
if !found {
fs.Debug(o, "Couldn't find object")
return fmt.Errorf("Couldn't find object")
}
return nil
}
// Path should be directory path either "" or "path/"
//
// List the directory using a recursive list from the root
//
// This fetches the minimum amount of stuff but does more API calls
// which makes it slow
func (f *Fs) listDirRecursive(dirID string, path string, out fs.ObjectsChan) error {
var subError error
// Make the API request
var wg sync.WaitGroup
_, err := f.listAll(dirID, false, false, func(info *api.Item) bool {
// Recurse on directories
if info.Folder != nil {
wg.Add(1)
folder := path + info.Name + "/"
fs.Debug(f, "Reading %s", folder)
go func() {
defer wg.Done()
err := f.listDirRecursive(info.ID, folder, out)
if err != nil {
subError = err
fs.ErrorLog(f, "Error reading %s:%s", folder, err)
}
}()
} else {
if fs := f.newObjectWithInfo(path+info.Name, info); fs != nil {
out <- fs
}
}
return false
})
wg.Wait()
fs.Debug(f, "Finished reading %s", path)
if err != nil {
return err
}
if subError != nil {
return subError
}
return nil
}
// seek to a new offset
//
// if reopen is true, then we won't attempt to use an io.Seeker interface
//
// Must be called with fh.mu held
func (fh *ReadFileHandle) seek(offset int64, reopen bool) error {
// Can we seek it directly?
oldReader := fh.r.GetReader()
if do, ok := oldReader.(io.Seeker); !reopen && ok {
fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d (io.Seeker)", fh.offset, offset)
_, err := do.Seek(offset, 0)
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read io.Seeker failed: %v", err)
return err
}
} else {
fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d", fh.offset, offset)
// if not re-open with a seek
r, err := fh.o.Open(&fs.SeekOption{Offset: offset})
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read seek failed: %v", err)
return err
}
err = oldReader.Close()
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read seek close old failed: %v", err)
}
// fh.r = fs.NewAccount(r, fh.o) // account the transfer
fh.r.UpdateReader(r)
}
fh.offset = offset
return nil
}
// ListDir reads the directory specified by the job into out, returning any more jobs
func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) {
fs.Debug(f, "Reading %q", job.Path)
_, err = f.listAll(job.DirID, "", false, false, func(item *drive.File) bool {
remote := job.Path + item.Title
switch {
case *driveAuthOwnerOnly && !isAuthOwned(item):
// ignore object or directory
case item.MimeType == driveFolderType:
if out.IncludeDirectory(remote) {
dir := &fs.Dir{
Name: remote,
Bytes: -1,
Count: -1,
}
dir.When, _ = time.Parse(timeFormatIn, item.ModifiedDate)
if out.AddDir(dir) {
return true
}
if job.Depth > 0 {
jobs = append(jobs, dircache.ListDirJob{DirID: item.Id, Path: remote + "/", Depth: job.Depth - 1})
}
}
case item.Md5Checksum != "":
// If item has MD5 sum it is a file stored on drive
o, err := f.newObjectWithInfo(remote, item)
if err != nil {
out.SetError(err)
return true
}
if out.Add(o) {
return true
}
case len(item.ExportLinks) != 0:
// If item has export links then it is a google doc
extension, link := f.findExportFormat(remote, item)
if extension == "" {
fs.Debug(remote, "No export formats found")
} else {
o, err := f.newObjectWithInfo(remote+"."+extension, item)
if err != nil {
out.SetError(err)
return true
}
obj := o.(*Object)
obj.isDocument = true
obj.url = link
obj.bytes = -1
if out.Add(o) {
return true
}
}
default:
fs.Debug(remote, "Ignoring unknown object")
}
return false
})
fs.Debug(f, "Finished reading %q", job.Path)
return jobs, err
}
// purge deletes all the files and directories
//
// if oldOnly is true then it deletes only non current files.
//
// Implemented here so we can make sure we delete old versions.
func (f *Fs) purge(oldOnly bool) error {
var errReturn error
var checkErrMutex sync.Mutex
var checkErr = func(err error) {
if err == nil {
return
}
checkErrMutex.Lock()
defer checkErrMutex.Unlock()
if errReturn == nil {
errReturn = err
}
}
// Delete Config.Transfers in parallel
toBeDeleted := make(chan *api.File, fs.Config.Transfers)
var wg sync.WaitGroup
wg.Add(fs.Config.Transfers)
for i := 0; i < fs.Config.Transfers; i++ {
go func() {
defer wg.Done()
for object := range toBeDeleted {
fs.Stats.Checking(object.Name)
checkErr(f.deleteByID(object.ID, object.Name))
fs.Stats.DoneChecking(object.Name)
}
}()
}
last := ""
checkErr(f.list("", fs.MaxLevel, "", 0, true, func(remote string, object *api.File, isDirectory bool) error {
if !isDirectory {
fs.Stats.Checking(remote)
if oldOnly && last != remote {
if object.Action == "hide" {
fs.Debug(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
toBeDeleted <- object
} else {
fs.Debug(remote, "Not deleting current version (id %q) %q", object.ID, object.Action)
}
} else {
fs.Debug(remote, "Deleting (id %q)", object.ID)
toBeDeleted <- object
}
last = remote
fs.Stats.DoneChecking(remote)
}
return nil
}))
close(toBeDeleted)
wg.Wait()
if !oldOnly {
checkErr(f.Rmdir(""))
}
return errReturn
}
// s3Connection makes a connection to s3
func s3Connection(name string) (*s3.S3, *session.Session, error) {
// Make the auth
accessKeyID := fs.ConfigFile.MustValue(name, "access_key_id")
secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
var auth *credentials.Credentials
switch {
case accessKeyID == "" && secretAccessKey == "":
fs.Debug(name, "Using anonymous access for S3")
auth = credentials.AnonymousCredentials
case accessKeyID == "":
return nil, nil, errors.New("access_key_id not found")
case secretAccessKey == "":
return nil, nil, errors.New("secret_access_key not found")
default:
auth = credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "")
}
endpoint := fs.ConfigFile.MustValue(name, "endpoint")
region := fs.ConfigFile.MustValue(name, "region")
if region == "" && endpoint == "" {
endpoint = "https://s3.amazonaws.com/"
}
if region == "" {
region = "us-east-1"
}
awsConfig := aws.NewConfig().
WithRegion(region).
WithMaxRetries(maxRetries).
WithCredentials(auth).
WithEndpoint(endpoint).
WithHTTPClient(fs.Config.Client()).
WithS3ForcePathStyle(true)
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
ses := session.New()
c := s3.New(ses, awsConfig)
if region == "other-v2-signature" {
fs.Debug(name, "Using v2 auth")
signer := func(req *request.Request) {
// Ignore AnonymousCredentials object
if req.Config.Credentials == credentials.AnonymousCredentials {
return
}
sign(accessKeyID, secretAccessKey, req.HTTPRequest)
}
c.Handlers.Sign.Clear()
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
c.Handlers.Sign.PushBack(signer)
}
// Add user agent
c.Handlers.Build.PushBack(func(r *request.Request) {
r.HTTPRequest.Header.Set("User-Agent", fs.UserAgent)
})
return c, ses, nil
}
// readMetaData gets the info if it hasn't already been fetched
func (o *FsObjectDropbox) readMetaData() (err error) {
if o.md5sum != "" {
return nil
}
// fs.Debug(o, "Reading metadata from datastore")
record, err := o.dropbox.readRecord(o.metadataKey())
if err != nil {
fs.Debug(o, "Couldn't read metadata: %s", err)
record = nil
}
if record != nil {
// Read md5sum
md5sumInterface, ok, err := record.Get(md5sumField)
if err != nil {
return err
}
if !ok {
fs.Debug(o, "Couldn't find md5sum in record")
} else {
md5sum, ok := md5sumInterface.(string)
if !ok {
fs.Debug(o, "md5sum not a string")
} else {
o.md5sum = md5sum
}
}
// read mtime
mtimeInterface, ok, err := record.Get(mtimeField)
if err != nil {
return err
}
if !ok {
fs.Debug(o, "Couldn't find mtime in record")
} else {
mtime, ok := mtimeInterface.(string)
if !ok {
fs.Debug(o, "mtime not a string")
} else {
modTime, err := time.Parse(timeFormatIn, mtime)
if err != nil {
return err
}
o.modTime = modTime
}
}
}
// Last resort
return o.readEntryAndSetMetadata()
}
// read the directory
func (d *Dir) readDir() error {
d.mu.Lock()
defer d.mu.Unlock()
when := time.Now()
if d.read.IsZero() {
fs.Debug(d.path, "Reading directory")
} else {
age := when.Sub(d.read)
if age < dirCacheTime {
return nil
}
fs.Debug(d.path, "Re-reading directory (%v old)", age)
}
objs, dirs, err := fs.NewLister().SetLevel(1).Start(d.f, d.path).GetAll()
if err == fs.ErrorDirNotFound {
// We treat directory not found as empty because we
// create directories on the fly
} else if err != nil {
return err
}
// NB when we re-read a directory after its cache has expired
// we drop the old files which should lead to correct
// behaviour but may not be very efficient.
// Keep a note of the previous contents of the directory
oldItems := d.items
// Cache the items by name
d.items = make(map[string]*DirEntry, len(objs)+len(dirs))
for _, obj := range objs {
name := path.Base(obj.Remote())
d.items[name] = &DirEntry{
o: obj,
node: nil,
}
}
for _, dir := range dirs {
name := path.Base(dir.Remote())
// Use old dir value if it exists
if oldItem, ok := oldItems[name]; ok {
if _, ok := oldItem.o.(*fs.Dir); ok {
d.items[name] = oldItem
continue
}
}
d.items[name] = &DirEntry{
o: dir,
node: nil,
}
}
d.read = when
return nil
}
// Sets the modification time of the local fs object
func (o *FsObjectLocal) SetModTime(modTime time.Time) {
err := os.Chtimes(o.path, modTime, modTime)
if err != nil {
fs.Debug(o, "Failed to set mtime on file: %s", err)
return
}
// Re-read metadata
err = o.lstat()
if err != nil {
fs.Debug(o, "Failed to stat: %s", err)
return
}
}
// Lookup looks up a specific entry in the receiver.
//
// Lookup should return a Node corresponding to the entry. If the
// name does not exist in the directory, Lookup should return ENOENT.
//
// Lookup need not to handle the names "." and "..".
func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fusefs.Node, err error) {
path := path.Join(d.path, req.Name)
fs.Debug(path, "Dir.Lookup")
item, err := d.lookupNode(req.Name)
if err != nil {
if err != fuse.ENOENT {
fs.ErrorLog(path, "Dir.Lookup error: %v", err)
}
return nil, err
}
fs.Debug(path, "Dir.Lookup OK")
return item.node, nil
}
// checkUpload checks to see if an error occurred after the file was
// completely uploaded.
//
// If it was then it waits for a while to see if the file really
// exists and is the right size and returns an updated info.
//
// If the file wasn't found or was the wrong size then it returns the
// original error.
//
// This is a workaround for Amazon sometimes returning
//
// * 408 REQUEST_TIMEOUT
// * 504 GATEWAY_TIMEOUT
// * 500 Internal server error
//
// At the end of large uploads. The speculation is that the timeout
// is waiting for the sha1 hashing to complete and the file may well
// be properly uploaded.
func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, inInfo *acd.File, inErr error, uploadTime time.Duration) (fixedError bool, info *acd.File, err error) {
// Return if no error - all is well
if inErr == nil {
return false, inInfo, inErr
}
// If not one of the errors we can fix return
// if resp == nil || resp.StatusCode != 408 && resp.StatusCode != 500 && resp.StatusCode != 504 {
// return false, inInfo, inErr
// }
// The HTTP status
httpStatus := "HTTP status UNKNOWN"
if resp != nil {
httpStatus = resp.Status
}
// check to see if we read to the end
buf := make([]byte, 1)
n, err := in.Read(buf)
if !(n == 0 && err == io.EOF) {
fs.Debug(src, "Upload error detected but didn't finish upload: %v (%q)", inErr, httpStatus)
return false, inInfo, inErr
}
// Don't wait for uploads - assume they will appear later
if *uploadWaitPerGB <= 0 {
fs.Debug(src, "Upload error detected but waiting disabled: %v (%q)", inErr, httpStatus)
return false, inInfo, inErr
}
// Time we should wait for the upload
uploadWaitPerByte := float64(*uploadWaitPerGB) / 1024 / 1024 / 1024
timeToWait := time.Duration(uploadWaitPerByte * float64(src.Size()))
const sleepTime = 5 * time.Second // sleep between tries
retries := int((timeToWait + sleepTime - 1) / sleepTime) // number of retries, rounded up
fs.Debug(src, "Error detected after finished upload - waiting to see if object was uploaded correctly: %v (%q)", inErr, httpStatus)
remote := src.Remote()
for i := 1; i <= retries; i++ {
o, err := f.NewObject(remote)
if err == fs.ErrorObjectNotFound {
fs.Debug(src, "Object not found - waiting (%d/%d)", i, retries)
} else if err != nil {
fs.Debug(src, "Object returned error - waiting (%d/%d): %v", i, retries, err)
} else {
if src.Size() == o.Size() {
fs.Debug(src, "Object found with correct size %d after waiting (%d/%d) - %v - returning with no error", src.Size(), i, retries, sleepTime*time.Duration(i-1))
info = &acd.File{
Node: o.(*Object).info,
}
return true, info, nil
}
fs.Debug(src, "Object found but wrong size %d vs %d - waiting (%d/%d)", src.Size(), o.Size(), i, retries)
}
time.Sleep(sleepTime)
}
fs.Debug(src, "Giving up waiting for object - returning original error: %v (%q)", inErr, httpStatus)
return false, inInfo, inErr
}
// Create makes a new file
func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fusefs.Node, fusefs.Handle, error) {
path := path.Join(d.path, req.Name)
fs.Debug(path, "Dir.Create")
src := newCreateInfo(d.f, path)
// This gets added to the directory when the file is written
file := newFile(d, nil)
fh, err := newWriteFileHandle(d, file, src)
if err != nil {
fs.ErrorLog(path, "Dir.Create error: %v", err)
return nil, nil, err
}
fs.Debug(path, "Dir.Create OK")
return file, fh, nil
}
// Mkdir creates a new directory
func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fusefs.Node, error) {
// We just pretend to have created the directory - rclone will
// actually create the directory if we write files into it
path := path.Join(d.path, req.Name)
fs.Debug(path, "Dir.Mkdir")
fsDir := &fs.Dir{
Name: path,
When: time.Now(),
}
dir := newDir(d.f, path)
d.addObject(fsDir, dir)
fs.Debug(path, "Dir.Mkdir OK")
return dir, nil
}
// End a call to the drive API
//
// Refresh the pace given an error that was returned. It returns a
// boolean as to whether the operation should be retried.
//
// See https://developers.google.com/drive/web/handle-errors
// http://stackoverflow.com/questions/18529524/403-rate-limit-after-only-1-insert-per-second
func (f *FsDrive) endCall(err error) bool {
again := false
oldSleepTime := f.sleepTime
if err == nil {
f.sleepTime = (f.sleepTime<<decayConstant - f.sleepTime) >> decayConstant
if f.sleepTime < minSleep {
f.sleepTime = minSleep
}
if f.sleepTime != oldSleepTime {
fs.Debug(f, "Reducing sleep to %v", f.sleepTime)
}
} else {
fs.Debug(f, "Error recived: %T %#v", err, err)
// Check for net error Timeout()
if x, ok := err.(interface {
Timeout() bool
}); ok && x.Timeout() {
again = true
}
// Check for net error Temporary()
if x, ok := err.(interface {
Temporary() bool
}); ok && x.Temporary() {
again = true
}
switch gerr := err.(type) {
case *googleapi.Error:
if gerr.Code >= 500 && gerr.Code < 600 {
// All 5xx errors should be retried
again = true
} else if len(gerr.Errors) > 0 {
reason := gerr.Errors[0].Reason
if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
again = true
}
}
}
}
if again {
f.sleepTime *= 2
if f.sleepTime > maxSleep {
f.sleepTime = maxSleep
}
if f.sleepTime != oldSleepTime {
fs.Debug(f, "Rate limited, increasing sleep to %v", f.sleepTime)
}
}
return again
}
// uploadMultipart uploads a file using multipart upload
func (o *Object) uploadMultipart(in io.Reader, size int64) (err error) {
if chunkSize%(320*1024) != 0 {
return fmt.Errorf("Chunk size %d is not a multiple of 320k", chunkSize)
}
// Create upload session
fs.Debug(o, "Starting multipart upload")
session, err := o.createUploadSession()
if err != nil {
return err
}
uploadURL := session.UploadURL
// Cancel the session if something went wrong
defer func() {
if err != nil {
fs.Debug(o, "Cancelling multipart upload")
cancelErr := o.cancelUploadSession(uploadURL)
if cancelErr != nil {
fs.Log(o, "Failed to cancel multipart upload: %v", err)
}
}
}()
// Upload the chunks
remaining := size
position := int64(0)
buf := make([]byte, int64(chunkSize))
for remaining > 0 {
n := int64(chunkSize)
if remaining < n {
n = remaining
buf = buf[:n]
}
_, err = io.ReadFull(in, buf)
if err != nil {
return err
}
fs.Debug(o, "Uploading segment %d/%d size %d", position, size, n)
err = o.uploadFragment(uploadURL, position, size, buf)
if err != nil {
return err
}
remaining -= n
position += n
}
return nil
}
// seek to a new offset
func (fh *ReadFileHandle) seek(offset int64) error {
fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d", fh.offset, offset)
r, err := fh.o.Open(&fs.SeekOption{Offset: offset})
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read seek failed: %v", err)
return err
}
err = fh.r.Close()
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read seek close old failed: %v", err)
}
fh.r = r
fh.offset = offset
return nil
}
// ListDir reads the directory specified by the job into out, returning any more jobs
func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) {
fs.Debug(f, "Reading %q", job.Path)
maxTries := fs.Config.LowLevelRetries
for tries := 1; tries <= maxTries; tries++ {
_, err = f.listAll(job.DirID, "", false, false, func(node *acd.Node) bool {
remote := job.Path + *node.Name
switch *node.Kind {
case folderKind:
if out.IncludeDirectory(remote) {
dir := &fs.Dir{
Name: remote,
Bytes: -1,
Count: -1,
}
dir.When, _ = time.Parse(timeFormat, *node.ModifiedDate) // FIXME
if out.AddDir(dir) {
return true
}
if job.Depth > 0 {
jobs = append(jobs, dircache.ListDirJob{DirID: *node.Id, Path: remote + "/", Depth: job.Depth - 1})
}
}
case fileKind:
o, err := f.newObjectWithInfo(remote, node)
if err != nil {
out.SetError(err)
return true
}
if out.Add(o) {
return true
}
default:
// ignore ASSET etc
}
return false
})
if fs.IsRetryError(err) {
fs.Debug(f, "Directory listing error for %q: %v - low level retry %d/%d", job.Path, err, tries, maxTries)
continue
}
if err != nil {
return nil, err
}
break
}
fs.Debug(f, "Finished reading %q", job.Path)
return jobs, err
}
// setMetaData sets the fs data from a storage.Object
func (o *FsObjectStorage) setMetaData(info *storage.Object) {
o.url = info.MediaLink
o.bytes = int64(info.Size)
// Read md5sum
md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
if err != nil {
fs.Log(o, "Bad MD5 decode: %v", err)
} else {
o.md5sum = hex.EncodeToString(md5sumData)
}
// read mtime out of metadata if available
mtimeString, ok := info.Metadata[metaMtime]
if ok {
modTime, err := time.Parse(timeFormatIn, mtimeString)
if err == nil {
o.modTime = modTime
return
} else {
fs.Debug(o, "Failed to read mtime from metadata: %s", err)
}
}
// Fallback to the Updated time
modTime, err := time.Parse(timeFormatIn, info.Updated)
if err != nil {
fs.Log(o, "Bad time decode: %v", err)
} else {
o.modTime = modTime
}
}
// initConfig is run by cobra after initialising the flags
func initConfig() {
// Log file output
if *logFile != "" {
f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
if err != nil {
log.Fatalf("Failed to open log file: %v", err)
}
_, err = f.Seek(0, os.SEEK_END)
if err != nil {
fs.ErrorLog(nil, "Failed to seek log file to end: %v", err)
}
log.SetOutput(f)
fs.DebugLogger.SetOutput(f)
redirectStderr(f)
}
// Load the rest of the config now we have started the logger
fs.LoadConfig()
// Write the args for debug purposes
fs.Debug("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
// Setup CPU profiling if desired
if *cpuProfile != "" {
fs.Log(nil, "Creating CPU profile %q\n", *cpuProfile)
f, err := os.Create(*cpuProfile)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
// Setup memory profiling if desired
if *memProfile != "" {
defer func() {
fs.Log(nil, "Saving Memory profile %q\n", *memProfile)
f, err := os.Create(*memProfile)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = pprof.WriteHeapProfile(f)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = f.Close()
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
}()
}
}
// Move src to this remote using server side move operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
// go test -v -run '^Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|FsMove)$'
srcObj, ok := src.(*Object)
if !ok {
fs.Debug(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
// create the destination directory if necessary
err := f.dirCache.FindRoot(true)
if err != nil {
return nil, err
}
srcLeaf, srcDirectoryID, err := srcObj.fs.dirCache.FindPath(srcObj.remote, false)
if err != nil {
return nil, err
}
dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(remote, true)
if err != nil {
return nil, err
}
dstInfo, err := f.moveNode(srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
if err != nil {
return nil, err
}
dstObj := &Object{
fs: f,
remote: remote,
info: dstInfo,
}
return dstObj, nil
}
// readMetaData gets the info if it hasn't already been fetched
func (o *FsObjectDrive) readMetaData() (err error) {
if o.id != "" {
return nil
}
leaf, directoryID, err := o.drive.dirCache.FindPath(o.remote, false)
if err != nil {
return err
}
found, err := o.drive.listAll(directoryID, leaf, false, true, func(item *drive.File) bool {
if item.Title == leaf {
o.setMetaData(item)
return true
}
return false
})
if err != nil {
return err
}
if !found {
fs.Debug(o, "Couldn't find object")
return fmt.Errorf("Couldn't find object")
}
return nil
}
// FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
//fs.Debug(f, "FindLeaf(%q, %q)", pathID, leaf)
folder := acd.FolderFromId(pathID, f.c.Nodes)
var resp *http.Response
var subFolder *acd.Folder
err = f.pacer.Call(func() (bool, error) {
subFolder, resp, err = folder.GetFolder(leaf)
return f.shouldRetry(resp, err)
})
if err != nil {
if err == acd.ErrorNodeNotFound {
//fs.Debug(f, "...Not found")
return "", false, nil
}
//fs.Debug(f, "...Error %v", err)
return "", false, err
}
if subFolder.Status != nil && *subFolder.Status != statusAvailable {
fs.Debug(f, "Ignoring folder %q in state %q", leaf, *subFolder.Status)
time.Sleep(1 * time.Second) // FIXME wait for problem to go away!
return "", false, nil
}
//fs.Debug(f, "...Found(%q, %v)", *subFolder.Id, leaf)
return *subFolder.Id, true, nil
}
// updateChunks updates the existing object using chunks to a separate
// container. It returns a string which prefixes current segments.
func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) (string, error) {
// Create the segmentsContainer if it doesn't exist
err := o.fs.c.ContainerCreate(o.fs.segmentsContainer, nil)
if err != nil {
return "", err
}
// Upload the chunks
left := size
i := 0
uniquePrefix := fmt.Sprintf("%s/%d", swift.TimeToFloatString(time.Now()), size)
segmentsPath := fmt.Sprintf("%s%s/%s", o.fs.root, o.remote, uniquePrefix)
for left > 0 {
n := min(left, int64(chunkSize))
headers["Content-Length"] = strconv.FormatInt(n, 10) // set Content-Length as we know it
segmentReader := io.LimitReader(in, n)
segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i)
fs.Debug(o, "Uploading segment file %q into %q", segmentPath, o.fs.segmentsContainer)
_, err := o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
if err != nil {
return "", err
}
left -= n
i++
}
// Upload the manifest
headers["X-Object-Manifest"] = fmt.Sprintf("%s/%s", o.fs.segmentsContainer, segmentsPath)
headers["Content-Length"] = "0" // set Content-Length as we know it
emptyReader := bytes.NewReader(nil)
manifestName := o.fs.root + o.remote
_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", "", headers)
return uniquePrefix + "/", err
}
// getCredentials reads the OpenStack Credentials using the Hubic API
//
// The credentials are read into the Fs
func (f *Fs) getCredentials() (err error) {
req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil)
if err != nil {
return err
}
req.Header.Add("User-Agent", fs.UserAgent)
resp, err := f.client.Do(req)
if err != nil {
return err
}
defer fs.CheckClose(resp.Body, &err)
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return errors.Errorf("failed to get credentials: %s", resp.Status)
}
decoder := json.NewDecoder(resp.Body)
var result credentials
err = decoder.Decode(&result)
if err != nil {
return err
}
// fs.Debug(f, "Got credentials %+v", result)
if result.Token == "" || result.Endpoint == "" || result.Expires == "" {
return errors.New("couldn't read token, result and expired from credentials")
}
f.credentials = result
expires, err := time.Parse(time.RFC3339, result.Expires)
if err != nil {
return err
}
f.expires = expires
fs.Debug(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now()))
return nil
}
// Move src to this remote using server side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
func (dstFs *FsDrive) DirMove(src fs.Fs) error {
srcFs, ok := src.(*FsDrive)
if !ok {
fs.Debug(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
// Check if destination exists
dstFs.dirCache.ResetRoot()
err := dstFs.dirCache.FindRoot(false)
if err == nil {
return fs.ErrorDirExists
}
// Find ID of parent
leaf, directoryId, err := dstFs.dirCache.FindPath(dstFs.root, true)
if err != nil {
return err
}
// Do the move
patch := drive.File{
Title: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}},
}
_, err = dstFs.svc.Files.Patch(srcFs.dirCache.RootID(), &patch).Do()
if err != nil {
return err
}
srcFs.dirCache.ResetRoot()
return nil
}
// DirMove moves src directory to this remote using server side move
// operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
func (f *Fs) DirMove(src fs.Fs) error {
srcFs, ok := src.(*Fs)
if !ok {
fs.Debug(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
// Check if source exists
sstat, err := os.Lstat(srcFs.root)
if err != nil {
return err
}
// And is a directory
if !sstat.IsDir() {
return fs.ErrorCantDirMove
}
// Check if destination exists
_, err = os.Lstat(f.root)
if !os.IsNotExist(err) {
return fs.ErrorDirExists
}
// Do the move
return os.Rename(srcFs.root, f.root)
}
// Mount mounts the remote at mountpoint.
//
// If noModTime is set then it
func Mount(f fs.Fs, mountpoint string) error {
if debugFUSE {
fuse.Debug = func(msg interface{}) {
fs.Debug("fuse", "%v", msg)
}
}
// Set permissions
dirPerms = 0777 &^ os.FileMode(umask)
filePerms = 0666 &^ os.FileMode(umask)
// Show stats if the user has specifically requested them
if cmd.ShowStats() {
stopStats := cmd.StartStats()
defer close(stopStats)
}
// Mount it
errChan, err := mount(f, mountpoint)
if err != nil {
return errors.Wrap(err, "failed to mount FUSE fs")
}
// Wait for umount
err = <-errChan
if err != nil {
return errors.Wrap(err, "failed to umount FUSE fs")
}
return nil
}