func doPass(sc, dc *client.Client, passNum int) (stats SyncStats, retErr error) {
srcBlobs := make(chan blobref.SizedBlobRef, 100)
destBlobs := make(chan blobref.SizedBlobRef, 100)
srcErr := make(chan error)
destErr := make(chan error)
go func() {
srcErr <- sc.EnumerateBlobs(srcBlobs)
}()
checkSourceError := func() {
if err := <-srcErr; err != nil {
retErr = errors.New(fmt.Sprintf("Enumerate error from source: %v", err))
}
}
if *flagDest == "stdout" {
for sb := range srcBlobs {
fmt.Printf("%s %d\n", sb.BlobRef, sb.Size)
}
checkSourceError()
return
}
go func() {
destErr <- dc.EnumerateBlobs(destBlobs)
}()
checkDestError := func() {
if err := <-destErr; err != nil {
retErr = errors.New(fmt.Sprintf("Enumerate error from destination: %v", err))
}
}
destNotHaveBlobs := make(chan blobref.SizedBlobRef, 100)
go client.ListMissingDestinationBlobs(destNotHaveBlobs, srcBlobs, destBlobs)
for sb := range destNotHaveBlobs {
fmt.Printf("Destination needs blob: %s\n", sb)
blobReader, size, err := sc.FetchStreaming(sb.BlobRef)
if err != nil {
stats.ErrorCount++
log.Printf("Error fetching %s: %v", sb.BlobRef, err)
continue
}
if size != sb.Size {
stats.ErrorCount++
log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
sb.Size, sb.BlobRef, size)
continue
}
uh := &client.UploadHandle{BlobRef: sb.BlobRef, Size: size, Contents: blobReader}
pr, err := dc.Upload(uh)
if err != nil {
stats.ErrorCount++
log.Printf("Upload of %s to destination blobserver failed: %v", sb.BlobRef, err)
continue
}
if !pr.Skipped {
stats.BlobsCopied++
stats.BytesCopied += pr.Size
}
if *flagRemoveSource {
if err = sc.RemoveBlob(sb.BlobRef); err != nil {
stats.ErrorCount++
log.Printf("Failed to delete %s from source: %v", sb.BlobRef, err)
}
}
}
checkSourceError()
checkDestError()
if retErr == nil && stats.ErrorCount > 0 {
retErr = errors.New(fmt.Sprintf("%d errors during sync", stats.ErrorCount))
}
return stats, retErr
}
// src: non-nil source
// dest: non-nil destination
// thirdLeg: optional third-leg client. if not nil, anything on src
// but not on dest will instead be copied to thirdLeg, instead of
// directly to dest. (sneakernet mode, copying to a portable drive
// and transporting thirdLeg to dest)
func (c *syncCmd) doPass(src, dest, thirdLeg blobserver.Storage) (stats SyncStats, retErr error) {
srcBlobs := make(chan blob.SizedRef, 100)
destBlobs := make(chan blob.SizedRef, 100)
srcErr := make(chan error, 1)
destErr := make(chan error, 1)
go func() {
srcErr <- enumerateAllBlobs(src, srcBlobs)
}()
checkSourceError := func() {
if err := <-srcErr; err != nil {
retErr = fmt.Errorf("Enumerate error from source: %v", err)
}
}
if c.dest == "stdout" {
for sb := range srcBlobs {
fmt.Printf("%s %d\n", sb.Ref, sb.Size)
}
checkSourceError()
return
}
go func() {
destErr <- enumerateAllBlobs(dest, destBlobs)
}()
checkDestError := func() {
if err := <-destErr; err != nil {
retErr = errors.New(fmt.Sprintf("Enumerate error from destination: %v", err))
}
}
destNotHaveBlobs := make(chan blob.SizedRef)
sizeMismatch := make(chan blob.Ref)
readSrcBlobs := srcBlobs
if c.verbose {
readSrcBlobs = loggingBlobRefChannel(srcBlobs)
}
mismatches := []blob.Ref{}
go client.ListMissingDestinationBlobs(destNotHaveBlobs, sizeMismatch, readSrcBlobs, destBlobs)
// Handle three-legged mode if tc is provided.
checkThirdError := func() {} // default nop
syncBlobs := destNotHaveBlobs
firstHopDest := dest
if thirdLeg != nil {
thirdBlobs := make(chan blob.SizedRef, 100)
thirdErr := make(chan error, 1)
go func() {
thirdErr <- enumerateAllBlobs(thirdLeg, thirdBlobs)
}()
checkThirdError = func() {
if err := <-thirdErr; err != nil {
retErr = fmt.Errorf("Enumerate error from third leg: %v", err)
}
}
thirdNeedBlobs := make(chan blob.SizedRef)
go client.ListMissingDestinationBlobs(thirdNeedBlobs, sizeMismatch, destNotHaveBlobs, thirdBlobs)
syncBlobs = thirdNeedBlobs
firstHopDest = thirdLeg
}
For:
for {
select {
case br := <-sizeMismatch:
// TODO(bradfitz): check both sides and repair, carefully. For now, fail.
log.Printf("WARNING: blobref %v has differing sizes on source and dest", br)
stats.ErrorCount++
mismatches = append(mismatches, br)
case sb, ok := <-syncBlobs:
if !ok {
break For
}
fmt.Printf("Destination needs blob: %s\n", sb)
blobReader, size, err := src.FetchStreaming(sb.Ref)
if err != nil {
stats.ErrorCount++
log.Printf("Error fetching %s: %v", sb.Ref, err)
continue
}
if size != sb.Size {
stats.ErrorCount++
log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
sb.Size, sb.Ref, size)
continue
}
if _, err := firstHopDest.ReceiveBlob(sb.Ref, blobReader); err != nil {
stats.ErrorCount++
log.Printf("Upload of %s to destination blobserver failed: %v", sb.Ref, err)
continue
}
stats.BlobsCopied++
stats.BytesCopied += size
if c.removeSrc {
if err = src.RemoveBlobs([]blob.Ref{sb.Ref}); err != nil {
stats.ErrorCount++
log.Printf("Failed to delete %s from source: %v", sb.Ref, err)
}
}
}
}
checkSourceError()
checkDestError()
checkThirdError()
if retErr == nil && stats.ErrorCount > 0 {
retErr = fmt.Errorf("%d errors during sync", stats.ErrorCount)
}
return stats, retErr
}
func doPass(sc, dc *client.Client, passNum int) (stats SyncStats, retErr error) {
srcBlobs := make(chan blobref.SizedBlobRef, 100)
destBlobs := make(chan blobref.SizedBlobRef, 100)
srcErr := make(chan error)
destErr := make(chan error)
go func() {
srcErr <- sc.SimpleEnumerateBlobs(srcBlobs)
}()
checkSourceError := func() {
if err := <-srcErr; err != nil {
retErr = fmt.Errorf("Enumerate error from source: %v", err)
}
}
if *flagDest == "stdout" {
for sb := range srcBlobs {
fmt.Printf("%s %d\n", sb.BlobRef, sb.Size)
}
checkSourceError()
return
}
go func() {
destErr <- dc.SimpleEnumerateBlobs(destBlobs)
}()
checkDestError := func() {
if err := <-destErr; err != nil {
retErr = errors.New(fmt.Sprintf("Enumerate error from destination: %v", err))
}
}
destNotHaveBlobs := make(chan blobref.SizedBlobRef)
sizeMismatch := make(chan *blobref.BlobRef)
readSrcBlobs := srcBlobs
if *flagVerbose {
readSrcBlobs = loggingBlobRefChannel(srcBlobs)
}
mismatches := []*blobref.BlobRef{}
go client.ListMissingDestinationBlobs(destNotHaveBlobs, sizeMismatch, readSrcBlobs, destBlobs)
For:
for {
select {
case br := <-sizeMismatch:
// TODO(bradfitz): check both sides and repair, carefully. For now, fail.
log.Printf("WARNING: blobref %v has differing sizes on source and est", br)
stats.ErrorCount++
mismatches = append(mismatches, br)
case sb, ok := <-destNotHaveBlobs:
if !ok {
break For
}
fmt.Printf("Destination needs blob: %s\n", sb)
blobReader, size, err := sc.FetchStreaming(sb.BlobRef)
if err != nil {
stats.ErrorCount++
log.Printf("Error fetching %s: %v", sb.BlobRef, err)
continue
}
if size != sb.Size {
stats.ErrorCount++
log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
sb.Size, sb.BlobRef, size)
continue
}
uh := &client.UploadHandle{BlobRef: sb.BlobRef, Size: size, Contents: blobReader}
pr, err := dc.Upload(uh)
if err != nil {
stats.ErrorCount++
log.Printf("Upload of %s to destination blobserver failed: %v", sb.BlobRef, err)
continue
}
if !pr.Skipped {
stats.BlobsCopied++
stats.BytesCopied += pr.Size
}
if *flagRemoveSource {
if err = sc.RemoveBlob(sb.BlobRef); err != nil {
stats.ErrorCount++
log.Printf("Failed to delete %s from source: %v", sb.BlobRef, err)
}
}
}
}
checkSourceError()
checkDestError()
if retErr == nil && stats.ErrorCount > 0 {
retErr = errors.New(fmt.Sprintf("%d errors during sync", stats.ErrorCount))
}
return stats, retErr
}