func doSign(output string, signature string, compression pwr.CompressionSettings, fixPerms bool) error { comm.Opf("Creating signature for %s", output) startTime := time.Now() container, err := tlc.WalkAny(output, filterPaths) if err != nil { return errors.Wrap(err, 1) } pool, err := pools.New(container, output) if err != nil { return errors.Wrap(err, 1) } if fixPerms { container.FixPermissions(pool) } signatureWriter, err := os.Create(signature) if err != nil { return errors.Wrap(err, 1) } rawSigWire := wire.NewWriteContext(signatureWriter) rawSigWire.WriteMagic(pwr.SignatureMagic) rawSigWire.WriteMessage(&pwr.SignatureHeader{ Compression: &compression, }) sigWire, err := pwr.CompressWire(rawSigWire, &compression) if err != nil { return errors.Wrap(err, 1) } sigWire.WriteMessage(container) comm.StartProgress() err = pwr.ComputeSignatureToWriter(container, pool, comm.NewStateConsumer(), func(hash wsync.BlockHash) error { return sigWire.WriteMessage(&pwr.BlockHash{ WeakHash: hash.WeakHash, StrongHash: hash.StrongHash, }) }) comm.EndProgress() if err != nil { return errors.Wrap(err, 1) } err = sigWire.Close() if err != nil { return errors.Wrap(err, 1) } prettySize := humanize.IBytes(uint64(container.Size)) perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds())) comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond) return nil }
func doWalk(path string, out chan walkResult, errs chan error, fixPerms bool) { container, err := tlc.WalkAny(path, filterPaths) if err != nil { errs <- errors.Wrap(err, 1) return } pool, err := pools.New(container, path) if err != nil { errs <- errors.Wrap(err, 1) return } result := walkResult{ container: container, pool: pool, } if fixPerms { result.container.FixPermissions(result.pool) } out <- result }
func doDiff(target string, source string, patch string, compression pwr.CompressionSettings) error { var err error startTime := time.Now() targetSignature := &pwr.SignatureInfo{} targetSignature.Container, err = tlc.WalkAny(target, filterPaths) if err != nil { // Signature file perhaps? var signatureReader io.ReadCloser signatureReader, err = eos.Open(target) if err != nil { return errors.Wrap(err, 1) } targetSignature, err = pwr.ReadSignature(signatureReader) if err != nil { if errors.Is(err, wire.ErrFormat) { return fmt.Errorf("unrecognized target %s (not a container, not a signature file)", target) } return errors.Wrap(err, 1) } comm.Opf("Read signature from %s", target) err = signatureReader.Close() if err != nil { return errors.Wrap(err, 1) } } else { // Container (dir, archive, etc.) comm.Opf("Hashing %s", target) comm.StartProgress() var targetPool wsync.Pool targetPool, err = pools.New(targetSignature.Container, target) if err != nil { return errors.Wrap(err, 1) } targetSignature.Hashes, err = pwr.ComputeSignature(targetSignature.Container, targetPool, comm.NewStateConsumer()) comm.EndProgress() if err != nil { return errors.Wrap(err, 1) } { prettySize := humanize.IBytes(uint64(targetSignature.Container.Size)) perSecond := humanize.IBytes(uint64(float64(targetSignature.Container.Size) / time.Since(startTime).Seconds())) comm.Statf("%s (%s) @ %s/s\n", prettySize, targetSignature.Container.Stats(), perSecond) } } startTime = time.Now() var sourceContainer *tlc.Container sourceContainer, err = tlc.WalkAny(source, filterPaths) if err != nil { return errors.Wrap(err, 1) } var sourcePool wsync.Pool sourcePool, err = pools.New(sourceContainer, source) if err != nil { return errors.Wrap(err, 1) } patchWriter, err := os.Create(patch) if err != nil { return errors.Wrap(err, 1) } defer patchWriter.Close() signaturePath := patch + ".sig" signatureWriter, err := os.Create(signaturePath) if err != nil { return errors.Wrap(err, 1) } defer signatureWriter.Close() patchCounter := counter.NewWriter(patchWriter) signatureCounter := counter.NewWriter(signatureWriter) dctx := &pwr.DiffContext{ SourceContainer: sourceContainer, Pool: sourcePool, TargetContainer: targetSignature.Container, TargetSignature: targetSignature.Hashes, Consumer: comm.NewStateConsumer(), Compression: &compression, } comm.Opf("Diffing %s", source) comm.StartProgress() err = dctx.WritePatch(patchCounter, signatureCounter) if err != nil { return errors.Wrap(err, 1) } comm.EndProgress() totalDuration := time.Since(startTime) { prettySize := humanize.IBytes(uint64(sourceContainer.Size)) perSecond := humanize.IBytes(uint64(float64(sourceContainer.Size) / totalDuration.Seconds())) comm.Statf("%s (%s) @ %s/s\n", prettySize, sourceContainer.Stats(), perSecond) } if *diffArgs.verify { tmpDir, err := ioutil.TempDir("", "pwr") if err != nil { return errors.Wrap(err, 1) } defer os.RemoveAll(tmpDir) apply(patch, target, tmpDir, false, signaturePath, "") } { prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count())) percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes) relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size) prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes)) comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize) comm.Statf("%s patch (%.2f%% of the full size) in %s", prettyPatchSize, relToNew, totalDuration) } return nil }
func (actx *ApplyContext) patchAll(patchWire *wire.ReadContext, signature *SignatureInfo) (retErr error) { sourceContainer := actx.SourceContainer var validatingPool *ValidatingPool consumerErrs := make(chan error, 1) outputPool := actx.OutputPool if outputPool == nil { outputPool = fspool.New(sourceContainer, actx.OutputPath) } if signature != nil { validatingPool = &ValidatingPool{ Pool: outputPool, Container: sourceContainer, Signature: signature, } if actx.WoundsPath != "" { validatingPool.Wounds = make(chan *Wound) actx.WoundsConsumer = &WoundsWriter{ WoundsPath: actx.WoundsPath, } } if actx.WoundsConsumer != nil { go func() { consumerErrs <- actx.WoundsConsumer.Do(signature.Container, validatingPool.Wounds) }() } outputPool = validatingPool } targetContainer := actx.TargetContainer targetPool := actx.TargetPool if targetPool == nil { if actx.TargetPath == "" { return fmt.Errorf("apply: need either TargetPool or TargetPath") } var cErr error targetPool, cErr = pools.New(targetContainer, actx.TargetPath) if cErr != nil { return cErr } } fileOffset := int64(0) sourceBytes := sourceContainer.Size onSourceWrite := func(count int64) { // we measure patching progress as the number of total bytes written // to the source container. no-ops (untouched files) count too, so the // progress bar may jump ahead a bit at times, but that's a good surprise // measuring progress by bytes of the patch read would just be a different // kind of inaccuracy (due to decompression buffers, etc.) actx.Consumer.Progress(float64(fileOffset+count) / float64(sourceBytes)) } sctx := mksync() sh := &SyncHeader{} // transpositions, indexed by TargetPath transpositions := make(map[string][]*Transposition) actx.transpositions = transpositions defer func() { var closeErr error closeErr = targetPool.Close() if closeErr != nil { if retErr == nil { retErr = errors.Wrap(closeErr, 1) } } closeErr = outputPool.Close() if closeErr != nil { if retErr == nil { retErr = errors.Wrap(closeErr, 1) } } if validatingPool != nil { if validatingPool.Wounds != nil { close(validatingPool.Wounds) } } if actx.WoundsConsumer != nil { taskErr := <-consumerErrs if taskErr != nil { if retErr == nil { retErr = errors.Wrap(taskErr, 1) } } } }() for fileIndex, f := range sourceContainer.Files { actx.Consumer.ProgressLabel(f.Path) actx.Consumer.Debug(f.Path) fileOffset = f.Offset // each series of patch operations is preceded by a SyncHeader giving // us the file index - it's a super basic measure to make sure the // patch file we're reading and the patching algorithm somewhat agree // on what's happening. sh.Reset() err := patchWire.ReadMessage(sh) if err != nil { retErr = errors.Wrap(err, 1) return } if sh.FileIndex != int64(fileIndex) { fmt.Printf("expected fileIndex = %d, got fileIndex %d\n", fileIndex, sh.FileIndex) retErr = errors.Wrap(ErrMalformedPatch, 1) return } ops := make(chan wsync.Operation) errc := make(chan error, 1) go readOps(patchWire, ops, errc) bytesWritten, transposition, err := actx.lazilyPatchFile(sctx, targetContainer, targetPool, sourceContainer, outputPool, sh.FileIndex, onSourceWrite, ops, actx.InPlace) if err != nil { select { case nestedErr := <-errc: if nestedErr != nil { actx.Consumer.Debugf("Had an error while reading ops: %s", nestedErr.Error()) } default: // no nested error } retErr = errors.Wrap(err, 1) return } if transposition != nil { transpositions[transposition.TargetPath] = append(transpositions[transposition.TargetPath], transposition) } else { actx.Stats.TouchedFiles++ if bytesWritten != f.Size { retErr = fmt.Errorf("%s: expected to write %d bytes, wrote %d bytes", f.Path, f.Size, bytesWritten) return } } // using errc to signal the end of processing, rather than having a separate // done channel. not sure if there's any upside to either err = <-errc if err != nil { retErr = err return } } err := actx.applyTranspositions(transpositions) if err != nil { retErr = err return } return }
func (vctx *ValidatorContext) validate(target string, signature *SignatureInfo, fileIndices chan int64, errs chan error, onProgress onProgressFunc, cancelled chan struct{}) { var retErr error targetPool, err := pools.New(signature.Container, target) if err != nil { errs <- err return } defer func() { err := targetPool.Close() if err != nil { retErr = errors.Wrap(err, 1) return } errs <- retErr }() aggregateOut := make(chan *Wound) relayDone := make(chan bool) go func() { for w := range aggregateOut { vctx.Wounds <- w } relayDone <- true }() wounds := AggregateWounds(aggregateOut, MaxWoundSize) defer func() { // signal no more wounds are going to be sent close(wounds) // wait for all of them to be relayed <-relayDone }() validatingPool := &ValidatingPool{ Pool: nullpool.New(signature.Container), Container: signature.Container, Signature: signature, Wounds: wounds, } doOne := func(fileIndex int64) error { file := signature.Container.Files[fileIndex] var reader io.Reader reader, err = targetPool.GetReader(fileIndex) if err != nil { if os.IsNotExist(err) { // whole file is missing wound := &Wound{ Kind: WoundKind_FILE, Index: fileIndex, Start: 0, End: file.Size, } onProgress(file.Size) select { case wounds <- wound: case <-cancelled: } return nil } return err } var writer io.WriteCloser writer, err = validatingPool.GetWriter(fileIndex) if err != nil { return err } defer writer.Close() lastCount := int64(0) countingWriter := counter.NewWriterCallback(func(count int64) { delta := count - lastCount onProgress(delta) lastCount = count }, writer) var writtenBytes int64 writtenBytes, err = io.Copy(countingWriter, reader) if err != nil { return err } if writtenBytes != file.Size { onProgress(file.Size - writtenBytes) wound := &Wound{ Kind: WoundKind_FILE, Index: fileIndex, Start: writtenBytes, End: file.Size, } select { case wounds <- wound: case <-cancelled: } } return nil } for { select { case fileIndex, ok := <-fileIndices: if !ok { // no more work return } err := doOne(fileIndex) if err != nil { if retErr == nil { retErr = err } return } case <-cancelled: // cancelled return } } }