// CheckMetadata downloads the metadata about all of the files currently // stored on Drive and compares it with the local cache. func (gd *GDrive) CheckMetadata(filename string, report func(string)) error { idToFile, err := gd.getIdToFile(filename) if err != nil { return err } // This will almost certainly take a while, so put up a progress bar. var bar *pb.ProgressBar if !gd.quiet { bar = pb.New(len(idToFile)) bar.ShowBar = true bar.ShowCounters = false bar.Output = os.Stderr bar.Prefix("Checking metadata cache: ") bar.Start() } err = gd.runQuery("trashed=false", func(f *drive.File) { if file, ok := idToFile[f.Id]; ok { df := newFile(f.Title, f) if !filesEqual(df, file) { report(fmt.Sprintf("%s: metadata mismatch.\nLocal: %+v\nDrive: %+v", file.Path, file, df)) } if bar != nil { bar.Increment() } delete(idToFile, f.Id) } else { // It'd be preferable to have "sharedWithMe=false" included in // the query string above, but the combination of that with // "trashed=false" seems to lead to no results being returned. if f.Shared == false { report(fmt.Sprintf("%s: found on Drive, not in local cache [%+v]", f.Title, f)) } } }) for _, f := range idToFile { report(fmt.Sprintf("%s: found in local cache, not on Drive [%+v]", f.Path, f)) } if bar != nil { bar.Finish() } return nil }
// Synchronize a local directory hierarchy with Google Drive. // localPath is the file or directory to start with, driveRoot is // the directory into which the file/directory will be sent func syncHierarchyUp(localPath string, driveRoot string, encrypt bool, trustTimes bool, maxSymlinkDepth int) int { if encrypt && key == nil { key = decryptEncryptionKey() } fileMappings, nUploadErrors := compileUploadFileTree(localPath, driveRoot, encrypt, trustTimes, maxSymlinkDepth) if len(fileMappings) == 0 { message("No files to be uploaded.") return 0 } nBytesToUpload := int64(0) for _, info := range fileMappings { if !info.LocalFileInfo.IsDir() { nBytesToUpload += info.LocalFileInfo.Size() } } // Given the list of files to sync, first find all of the directories and // then either get or create a Drive folder for each one. directoryMappingMap := make(map[string]localToRemoteFileMapping) var directoryNames []string for _, localfile := range fileMappings { if localfile.LocalFileInfo.IsDir() { directoryNames = append(directoryNames, localfile.DrivePath) directoryMappingMap[localfile.DrivePath] = localfile } } // Now sort the directories by name, which ensures that the parent of each // directory has already been created if we need to create its children. sort.Strings(directoryNames) if len(directoryNames) > 0 { // Actually create/update the directories. var dirProgressBar *pb.ProgressBar if !quiet { dirProgressBar = pb.New(len(directoryNames)) dirProgressBar.Output = os.Stderr dirProgressBar.Prefix("Directories: ") dirProgressBar.Start() } // Sync each of the directories, which serves to create any missing ones. for _, dirName := range directoryNames { file := directoryMappingMap[dirName] err := syncFileUp(file.LocalPath, file.LocalFileInfo, file.DrivePath, encrypt, dirProgressBar) if err != nil { // Errors creating directories are basically unrecoverable, // as they'll prevent us from later uploading any files in // them. printErrorAndExit(err) } } if dirProgressBar != nil { dirProgressBar.Finish() } } var fileProgressBar *pb.ProgressBar if !quiet { fileProgressBar = pb.New64(nBytesToUpload).SetUnits(pb.U_BYTES) fileProgressBar.Output = os.Stderr fileProgressBar.Prefix("Files: ") fileProgressBar.Start() } // Sort the files by size, small to large. sort.Sort(localToRemoteBySize(fileMappings)) // The two indices uploadFrontIndex and uploadBackIndex point to the // range of elements in the fileMappings array that haven't yet been // uploaded. uploadFrontIndex := 0 uploadBackIndex := len(fileMappings) - 1 // First, upload any large files that will use the resumable upload // protocol using a single thread; more threads here doesn't generally // help improve bandwidth utilizaiton and seems to make rate limit // errors from the Drive API more frequent... for ; uploadBackIndex >= 0; uploadBackIndex-- { if fileMappings[uploadBackIndex].LocalFileInfo.Size() < resumableUploadMinSize { break } fm := fileMappings[uploadBackIndex] if fm.LocalFileInfo.IsDir() { continue } if err := syncFileUp(fm.LocalPath, fm.LocalFileInfo, fm.DrivePath, encrypt, fileProgressBar); err != nil { addErrorAndPrintMessage(&nUploadErrors, fm.LocalPath, err) } } // Upload worker threads send a value over this channel when // they're done; the code that launches them waits for all of them // to do so before returning. doneChan := make(chan int, nWorkers) // Now that multiple threads are running, we need a mutex to protect // access to uploadFrontIndex and uploadBackIndex. var uploadIndexMutex sync.Mutex // All but one of the upload threads will grab files to upload starting // from the begining of the fileMappings array, thus doing the smallest // files first; one thread starts from the back of the array, doing the // largest files first. In this way, the large files help saturate the // available upload bandwidth and hide the fixed overhead of creating // the smaller files. uploadWorker := func(startFromFront bool) { for { uploadIndexMutex.Lock() if uploadFrontIndex > uploadBackIndex { // All files have been uploaded. debug.Printf("All files uploaded [%d,%d]; exiting", uploadFrontIndex, uploadBackIndex) uploadIndexMutex.Unlock() doneChan <- 1 break } // Get the index into fileMappings for the next file this // worker should upload. var index int if startFromFront { index = uploadFrontIndex uploadFrontIndex++ } else { index = uploadBackIndex uploadBackIndex-- } uploadIndexMutex.Unlock() fm := fileMappings[index] if fm.LocalFileInfo.IsDir() { // Directories have already been taken care of. continue } err := syncFileUp(fm.LocalPath, fm.LocalFileInfo, fm.DrivePath, encrypt, fileProgressBar) if err != nil { atomic.AddInt32(&nUploadErrors, 1) fmt.Fprintf(os.Stderr, "\nskicka: %s: %v\n", fm.LocalPath, err) } } } // Launch the workers. for i := 0; i < nWorkers; i++ { // All workers except the first one start from the front of // the array. go uploadWorker(i != 0) } // Wait for all of the workers to finish. for i := 0; i < nWorkers; i++ { <-doneChan } if fileProgressBar != nil { fileProgressBar.Finish() } if nUploadErrors > 0 { fmt.Fprintf(os.Stderr, "skicka: %d files not uploaded due to errors. "+ "This may be a transient failure; try uploading again.\n", nUploadErrors) } return int(nUploadErrors) }
func (gd *GDrive) getMetadataChanges(svc *drive.Service, startChangeId int64, changeChan chan<- []*drive.Change, errorChan chan<- error) { var about *drive.About var err error // Get the Drive About information in order to figure out how many // changes we need to download to get up to date. for try := 0; ; try++ { about, err = svc.About.Get().Do() if err == nil { break } else { err = gd.tryToHandleDriveAPIError(err, try) } if err != nil { errorChan <- err return } } // Don't clutter the output with a progress bar unless it looks like // downloading changes may take a while. // TODO: consider using timer.AfterFunc to put up the progress bar if // we're not done after a few seconds? It's not clear if this is worth // the trouble. var bar *pb.ProgressBar numChanges := about.LargestChangeId - startChangeId if numChanges > 1000 && !gd.quiet { bar = pb.New64(numChanges) bar.ShowBar = true bar.ShowCounters = false bar.Output = os.Stderr bar.Prefix("Updating metadata cache: ") bar.Start() } pageToken := "" try := 0 // Keep asking Drive for more changes until we get through them all. for { // Only ask for the fields in the drive.Change structure that we // actually to be filled in to save some bandwidth... fields := []googleapi.Field{"nextPageToken", "items/id", "items/fileId", "items/deleted", "items/file/id", "items/file/parents", "items/file/title", "items/file/fileSize", "items/file/mimeType", "items/file/properties", "items/file/modifiedDate", "items/file/md5Checksum", "items/file/labels"} q := svc.Changes.List().MaxResults(1000).IncludeSubscribed(false).Fields(fields...) if startChangeId >= 0 { q = q.StartChangeId(startChangeId + 1) } if pageToken != "" { q = q.PageToken(pageToken) } r, err := q.Do() if err != nil { err = gd.tryToHandleDriveAPIError(err, try) if err != nil { errorChan <- err return } try++ continue } // Success. Reset the try counter in case we had errors leading up // to this. try = 0 if len(r.Items) > 0 { // Send the changes along to the goroutine that's updating the // local cache. changeChan <- r.Items if bar != nil { bar.Set(int(r.Items[len(r.Items)-1].Id - startChangeId)) } } pageToken = string(r.NextPageToken) if pageToken == "" { break } } // Signal that no more changes are coming. close(changeChan) if bar != nil { bar.Finish() } gd.debug("Done updating metadata from Drive") }