func (sh *SyncHandler) validateShardPrefix(pfx string) (err error) {
defer func() {
sh.mu.Lock()
if err != nil {
errs := fmt.Sprintf("Failed to validate prefix %s: %v", pfx, err)
sh.logf("%s", errs)
sh.vshardErrs = append(sh.vshardErrs, errs)
} else {
sh.vshardDone++
}
sh.mu.Unlock()
}()
ctx := context.New()
defer ctx.Cancel()
src, serrc := sh.startValidatePrefix(ctx, pfx, false)
dst, derrc := sh.startValidatePrefix(ctx, pfx, true)
missing := make(chan blob.SizedRef, 8)
go blobserver.ListMissingDestinationBlobs(missing, func(blob.Ref) {}, src, dst)
for sb := range missing {
sh.mu.Lock()
sh.vmissing++
sh.mu.Unlock()
// TODO: stats for missing blobs found.
sh.enqueue(sb)
}
if err := <-serrc; err != nil {
return fmt.Errorf("Error enumerating source %s for validating shard %s: %v", sh.fromName, pfx, err)
}
if err := <-derrc; err != nil {
return fmt.Errorf("Error enumerating target %s for validating shard %s: %v", sh.toName, pfx, err)
}
return nil
}
func (sh *SyncHandler) validateShardPrefix(pfx string) (err error) {
defer func() {
sh.mu.Lock()
if err != nil {
errs := fmt.Sprintf("Failed to validate prefix %s: %v", pfx, err)
sh.logf("%s", errs)
sh.vshardErrs = append(sh.vshardErrs, errs)
} else {
sh.vshardDone++
}
sh.mu.Unlock()
}()
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
src, serrc := sh.startValidatePrefix(ctx, pfx, false)
dst, derrc := sh.startValidatePrefix(ctx, pfx, true)
srcErr := &chanError{
C: serrc,
Wrap: func(err error) error {
return fmt.Errorf("Error enumerating source %s for validating shard %s: %v", sh.fromName, pfx, err)
},
}
dstErr := &chanError{
C: derrc,
Wrap: func(err error) error {
return fmt.Errorf("Error enumerating target %s for validating shard %s: %v", sh.toName, pfx, err)
},
}
missingc := make(chan blob.SizedRef, 8)
go blobserver.ListMissingDestinationBlobs(missingc, func(blob.Ref) {}, src, dst)
var missing []blob.SizedRef
for sb := range missingc {
missing = append(missing, sb)
}
if err := srcErr.Get(); err != nil {
return err
}
if err := dstErr.Get(); err != nil {
return err
}
for _, sb := range missing {
if enqErr := sh.enqueue(sb); enqErr != nil {
if err == nil {
err = enqErr
}
} else {
sh.mu.Lock()
sh.vmissing += 1
sh.mu.Unlock()
}
}
return err
}
// src: non-nil source
// dest: non-nil destination
// thirdLeg: optional third-leg client. if not nil, anything on src
// but not on dest will instead be copied to thirdLeg, instead of
// directly to dest. (sneakernet mode, copying to a portable drive
// and transporting thirdLeg to dest)
func (c *syncCmd) doPass(src, dest, thirdLeg blobserver.Storage) (stats SyncStats, retErr error) {
var statsMu sync.Mutex // guards stats return value
srcBlobs := make(chan blob.SizedRef, 100)
destBlobs := make(chan blob.SizedRef, 100)
srcErr := make(chan error, 1)
destErr := make(chan error, 1)
ctx := context.TODO()
enumCtx, cancel := context.WithCancel(ctx) // used for all (2 or 3) enumerates
defer cancel()
enumerate := func(errc chan<- error, sto blobserver.Storage, blobc chan<- blob.SizedRef) {
err := enumerateAllBlobs(enumCtx, sto, blobc)
if err != nil {
cancel()
}
errc <- err
}
go enumerate(srcErr, src, srcBlobs)
checkSourceError := func() {
if err := <-srcErr; err != nil && err != context.Canceled {
retErr = fmt.Errorf("Enumerate error from source: %v", err)
}
}
if c.dest == "stdout" {
for sb := range srcBlobs {
fmt.Fprintf(cmdmain.Stdout, "%s %d\n", sb.Ref, sb.Size)
}
checkSourceError()
return
}
if c.wipe {
// TODO(mpl): dest is a client. make it send a "wipe" request?
// upon reception its server then wipes itself if it is a wiper.
log.Print("Index wiping not yet supported.")
}
go enumerate(destErr, dest, destBlobs)
checkDestError := func() {
if err := <-destErr; err != nil && err != context.Canceled {
retErr = fmt.Errorf("Enumerate error from destination: %v", err)
}
}
destNotHaveBlobs := make(chan blob.SizedRef)
readSrcBlobs := srcBlobs
if c.verbose {
readSrcBlobs = loggingBlobRefChannel(srcBlobs)
}
mismatches := []blob.Ref{}
logErrorf := func(format string, args ...interface{}) {
log.Printf(format, args...)
statsMu.Lock()
stats.ErrorCount++
statsMu.Unlock()
}
onMismatch := func(br blob.Ref) {
// TODO(bradfitz): check both sides and repair, carefully. For now, fail.
logErrorf("WARNING: blobref %v has differing sizes on source and dest", br)
mismatches = append(mismatches, br)
}
go blobserver.ListMissingDestinationBlobs(destNotHaveBlobs, onMismatch, readSrcBlobs, destBlobs)
// Handle three-legged mode if tc is provided.
checkThirdError := func() {} // default nop
syncBlobs := destNotHaveBlobs
firstHopDest := dest
if thirdLeg != nil {
thirdBlobs := make(chan blob.SizedRef, 100)
thirdErr := make(chan error, 1)
go enumerate(thirdErr, thirdLeg, thirdBlobs)
checkThirdError = func() {
if err := <-thirdErr; err != nil && err != context.Canceled {
retErr = fmt.Errorf("Enumerate error from third leg: %v", err)
}
}
thirdNeedBlobs := make(chan blob.SizedRef)
go blobserver.ListMissingDestinationBlobs(thirdNeedBlobs, onMismatch, destNotHaveBlobs, thirdBlobs)
syncBlobs = thirdNeedBlobs
firstHopDest = thirdLeg
}
var gate = syncutil.NewGate(c.concurrency)
var wg sync.WaitGroup
for sb := range syncBlobs {
sb := sb
gate.Start()
wg.Add(1)
go func() {
defer wg.Done()
defer gate.Done()
fmt.Fprintf(cmdmain.Stdout, "Destination needs blob: %s\n", sb)
blobReader, size, err := src.Fetch(sb.Ref)
if err != nil {
logErrorf("Error fetching %s: %v", sb.Ref, err)
return
}
if size != sb.Size {
logErrorf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
sb.Size, sb.Ref, size)
return
}
_, err = blobserver.Receive(firstHopDest, sb.Ref, blobReader)
if err != nil {
logErrorf("Upload of %s to destination blobserver failed: %v", sb.Ref, err)
return
}
statsMu.Lock()
stats.BlobsCopied++
stats.BytesCopied += int64(size)
statsMu.Unlock()
if c.removeSrc {
if err := src.RemoveBlobs([]blob.Ref{sb.Ref}); err != nil {
logErrorf("Failed to delete %s from source: %v", sb.Ref, err)
}
}
}()
}
wg.Wait()
checkSourceError()
checkDestError()
checkThirdError()
if retErr == nil && stats.ErrorCount > 0 {
retErr = fmt.Errorf("%d errors during sync", stats.ErrorCount)
}
return stats, retErr
}
// src: non-nil source
// dest: non-nil destination
// thirdLeg: optional third-leg client. if not nil, anything on src
// but not on dest will instead be copied to thirdLeg, instead of
// directly to dest. (sneakernet mode, copying to a portable drive
// and transporting thirdLeg to dest)
func (c *syncCmd) doPass(src, dest, thirdLeg blobserver.Storage) (stats SyncStats, retErr error) {
srcBlobs := make(chan blob.SizedRef, 100)
destBlobs := make(chan blob.SizedRef, 100)
srcErr := make(chan error, 1)
destErr := make(chan error, 1)
ctx := context.TODO()
defer ctx.Cancel()
go func() {
srcErr <- enumerateAllBlobs(ctx, src, srcBlobs)
}()
checkSourceError := func() {
if err := <-srcErr; err != nil {
retErr = fmt.Errorf("Enumerate error from source: %v", err)
}
}
if c.dest == "stdout" {
for sb := range srcBlobs {
fmt.Fprintf(cmdmain.Stdout, "%s %d\n", sb.Ref, sb.Size)
}
checkSourceError()
return
}
if c.wipe {
// TODO(mpl): dest is a client. make it send a "wipe" request?
// upon reception its server then wipes itself if it is a wiper.
log.Print("Index wiping not yet supported.")
}
go func() {
destErr <- enumerateAllBlobs(ctx, dest, destBlobs)
}()
checkDestError := func() {
if err := <-destErr; err != nil {
retErr = fmt.Errorf("Enumerate error from destination: %v", err)
}
}
destNotHaveBlobs := make(chan blob.SizedRef)
readSrcBlobs := srcBlobs
if c.verbose {
readSrcBlobs = loggingBlobRefChannel(srcBlobs)
}
mismatches := []blob.Ref{}
onMismatch := func(br blob.Ref) {
// TODO(bradfitz): check both sides and repair, carefully. For now, fail.
log.Printf("WARNING: blobref %v has differing sizes on source and dest", br)
stats.ErrorCount++
mismatches = append(mismatches, br)
}
go blobserver.ListMissingDestinationBlobs(destNotHaveBlobs, onMismatch, readSrcBlobs, destBlobs)
// Handle three-legged mode if tc is provided.
checkThirdError := func() {} // default nop
syncBlobs := destNotHaveBlobs
firstHopDest := dest
if thirdLeg != nil {
thirdBlobs := make(chan blob.SizedRef, 100)
thirdErr := make(chan error, 1)
go func() {
thirdErr <- enumerateAllBlobs(ctx, thirdLeg, thirdBlobs)
}()
checkThirdError = func() {
if err := <-thirdErr; err != nil {
retErr = fmt.Errorf("Enumerate error from third leg: %v", err)
}
}
thirdNeedBlobs := make(chan blob.SizedRef)
go blobserver.ListMissingDestinationBlobs(thirdNeedBlobs, onMismatch, destNotHaveBlobs, thirdBlobs)
syncBlobs = thirdNeedBlobs
firstHopDest = thirdLeg
}
for sb := range syncBlobs {
fmt.Fprintf(cmdmain.Stdout, "Destination needs blob: %s\n", sb)
blobReader, size, err := src.FetchStreaming(sb.Ref)
if err != nil {
stats.ErrorCount++
log.Printf("Error fetching %s: %v", sb.Ref, err)
continue
}
if size != sb.Size {
stats.ErrorCount++
log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
sb.Size, sb.Ref, size)
continue
}
if _, err := blobserver.Receive(firstHopDest, sb.Ref, blobReader); err != nil {
stats.ErrorCount++
log.Printf("Upload of %s to destination blobserver failed: %v", sb.Ref, err)
continue
}
stats.BlobsCopied++
stats.BytesCopied += int64(size)
if c.removeSrc {
if err = src.RemoveBlobs([]blob.Ref{sb.Ref}); err != nil {
stats.ErrorCount++
log.Printf("Failed to delete %s from source: %v", sb.Ref, err)
}
}
}
checkSourceError()
checkDestError()
checkThirdError()
if retErr == nil && stats.ErrorCount > 0 {
retErr = fmt.Errorf("%d errors during sync", stats.ErrorCount)
}
return stats, retErr
}