func doPass(sc, dc *client.Client, passNum int) (stats SyncStats, retErr error) { srcBlobs := make(chan blobref.SizedBlobRef, 100) destBlobs := make(chan blobref.SizedBlobRef, 100) srcErr := make(chan error) destErr := make(chan error) go func() { srcErr <- sc.EnumerateBlobs(srcBlobs) }() checkSourceError := func() { if err := <-srcErr; err != nil { retErr = errors.New(fmt.Sprintf("Enumerate error from source: %v", err)) } } if *flagDest == "stdout" { for sb := range srcBlobs { fmt.Printf("%s %d\n", sb.BlobRef, sb.Size) } checkSourceError() return } go func() { destErr <- dc.EnumerateBlobs(destBlobs) }() checkDestError := func() { if err := <-destErr; err != nil { retErr = errors.New(fmt.Sprintf("Enumerate error from destination: %v", err)) } } destNotHaveBlobs := make(chan blobref.SizedBlobRef, 100) go client.ListMissingDestinationBlobs(destNotHaveBlobs, srcBlobs, destBlobs) for sb := range destNotHaveBlobs { fmt.Printf("Destination needs blob: %s\n", sb) blobReader, size, err := sc.FetchStreaming(sb.BlobRef) if err != nil { stats.ErrorCount++ log.Printf("Error fetching %s: %v", sb.BlobRef, err) continue } if size != sb.Size { stats.ErrorCount++ log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d", sb.Size, sb.BlobRef, size) continue } uh := &client.UploadHandle{BlobRef: sb.BlobRef, Size: size, Contents: blobReader} pr, err := dc.Upload(uh) if err != nil { stats.ErrorCount++ log.Printf("Upload of %s to destination blobserver failed: %v", sb.BlobRef, err) continue } if !pr.Skipped { stats.BlobsCopied++ stats.BytesCopied += pr.Size } if *flagRemoveSource { if err = sc.RemoveBlob(sb.BlobRef); err != nil { stats.ErrorCount++ log.Printf("Failed to delete %s from source: %v", sb.BlobRef, err) } } } checkSourceError() checkDestError() if retErr == nil && stats.ErrorCount > 0 { retErr = errors.New(fmt.Sprintf("%d errors during sync", stats.ErrorCount)) } return stats, retErr }
// src: non-nil source // dest: non-nil destination // thirdLeg: optional third-leg client. if not nil, anything on src // but not on dest will instead be copied to thirdLeg, instead of // directly to dest. (sneakernet mode, copying to a portable drive // and transporting thirdLeg to dest) func (c *syncCmd) doPass(src, dest, thirdLeg blobserver.Storage) (stats SyncStats, retErr error) { srcBlobs := make(chan blob.SizedRef, 100) destBlobs := make(chan blob.SizedRef, 100) srcErr := make(chan error, 1) destErr := make(chan error, 1) go func() { srcErr <- enumerateAllBlobs(src, srcBlobs) }() checkSourceError := func() { if err := <-srcErr; err != nil { retErr = fmt.Errorf("Enumerate error from source: %v", err) } } if c.dest == "stdout" { for sb := range srcBlobs { fmt.Printf("%s %d\n", sb.Ref, sb.Size) } checkSourceError() return } go func() { destErr <- enumerateAllBlobs(dest, destBlobs) }() checkDestError := func() { if err := <-destErr; err != nil { retErr = errors.New(fmt.Sprintf("Enumerate error from destination: %v", err)) } } destNotHaveBlobs := make(chan blob.SizedRef) sizeMismatch := make(chan blob.Ref) readSrcBlobs := srcBlobs if c.verbose { readSrcBlobs = loggingBlobRefChannel(srcBlobs) } mismatches := []blob.Ref{} go client.ListMissingDestinationBlobs(destNotHaveBlobs, sizeMismatch, readSrcBlobs, destBlobs) // Handle three-legged mode if tc is provided. checkThirdError := func() {} // default nop syncBlobs := destNotHaveBlobs firstHopDest := dest if thirdLeg != nil { thirdBlobs := make(chan blob.SizedRef, 100) thirdErr := make(chan error, 1) go func() { thirdErr <- enumerateAllBlobs(thirdLeg, thirdBlobs) }() checkThirdError = func() { if err := <-thirdErr; err != nil { retErr = fmt.Errorf("Enumerate error from third leg: %v", err) } } thirdNeedBlobs := make(chan blob.SizedRef) go client.ListMissingDestinationBlobs(thirdNeedBlobs, sizeMismatch, destNotHaveBlobs, thirdBlobs) syncBlobs = thirdNeedBlobs firstHopDest = thirdLeg } For: for { select { case br := <-sizeMismatch: // TODO(bradfitz): check both sides and repair, carefully. For now, fail. log.Printf("WARNING: blobref %v has differing sizes on source and dest", br) stats.ErrorCount++ mismatches = append(mismatches, br) case sb, ok := <-syncBlobs: if !ok { break For } fmt.Printf("Destination needs blob: %s\n", sb) blobReader, size, err := src.FetchStreaming(sb.Ref) if err != nil { stats.ErrorCount++ log.Printf("Error fetching %s: %v", sb.Ref, err) continue } if size != sb.Size { stats.ErrorCount++ log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d", sb.Size, sb.Ref, size) continue } if _, err := firstHopDest.ReceiveBlob(sb.Ref, blobReader); err != nil { stats.ErrorCount++ log.Printf("Upload of %s to destination blobserver failed: %v", sb.Ref, err) continue } stats.BlobsCopied++ stats.BytesCopied += size if c.removeSrc { if err = src.RemoveBlobs([]blob.Ref{sb.Ref}); err != nil { stats.ErrorCount++ log.Printf("Failed to delete %s from source: %v", sb.Ref, err) } } } } checkSourceError() checkDestError() checkThirdError() if retErr == nil && stats.ErrorCount > 0 { retErr = fmt.Errorf("%d errors during sync", stats.ErrorCount) } return stats, retErr }
func doPass(sc, dc *client.Client, passNum int) (stats SyncStats, retErr error) { srcBlobs := make(chan blobref.SizedBlobRef, 100) destBlobs := make(chan blobref.SizedBlobRef, 100) srcErr := make(chan error) destErr := make(chan error) go func() { srcErr <- sc.SimpleEnumerateBlobs(srcBlobs) }() checkSourceError := func() { if err := <-srcErr; err != nil { retErr = fmt.Errorf("Enumerate error from source: %v", err) } } if *flagDest == "stdout" { for sb := range srcBlobs { fmt.Printf("%s %d\n", sb.BlobRef, sb.Size) } checkSourceError() return } go func() { destErr <- dc.SimpleEnumerateBlobs(destBlobs) }() checkDestError := func() { if err := <-destErr; err != nil { retErr = errors.New(fmt.Sprintf("Enumerate error from destination: %v", err)) } } destNotHaveBlobs := make(chan blobref.SizedBlobRef) sizeMismatch := make(chan *blobref.BlobRef) readSrcBlobs := srcBlobs if *flagVerbose { readSrcBlobs = loggingBlobRefChannel(srcBlobs) } mismatches := []*blobref.BlobRef{} go client.ListMissingDestinationBlobs(destNotHaveBlobs, sizeMismatch, readSrcBlobs, destBlobs) For: for { select { case br := <-sizeMismatch: // TODO(bradfitz): check both sides and repair, carefully. For now, fail. log.Printf("WARNING: blobref %v has differing sizes on source and est", br) stats.ErrorCount++ mismatches = append(mismatches, br) case sb, ok := <-destNotHaveBlobs: if !ok { break For } fmt.Printf("Destination needs blob: %s\n", sb) blobReader, size, err := sc.FetchStreaming(sb.BlobRef) if err != nil { stats.ErrorCount++ log.Printf("Error fetching %s: %v", sb.BlobRef, err) continue } if size != sb.Size { stats.ErrorCount++ log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d", sb.Size, sb.BlobRef, size) continue } uh := &client.UploadHandle{BlobRef: sb.BlobRef, Size: size, Contents: blobReader} pr, err := dc.Upload(uh) if err != nil { stats.ErrorCount++ log.Printf("Upload of %s to destination blobserver failed: %v", sb.BlobRef, err) continue } if !pr.Skipped { stats.BlobsCopied++ stats.BytesCopied += pr.Size } if *flagRemoveSource { if err = sc.RemoveBlob(sb.BlobRef); err != nil { stats.ErrorCount++ log.Printf("Failed to delete %s from source: %v", sb.BlobRef, err) } } } } checkSourceError() checkDestError() if retErr == nil && stats.ErrorCount > 0 { retErr = errors.New(fmt.Sprintf("%d errors during sync", stats.ErrorCount)) } return stats, retErr }