func doSign(output string, signature string, compression pwr.CompressionSettings, fixPerms bool) error {
comm.Opf("Creating signature for %s", output)
startTime := time.Now()
container, err := tlc.WalkAny(output, filterPaths)
if err != nil {
return errors.Wrap(err, 1)
}
pool, err := pools.New(container, output)
if err != nil {
return errors.Wrap(err, 1)
}
if fixPerms {
container.FixPermissions(pool)
}
signatureWriter, err := os.Create(signature)
if err != nil {
return errors.Wrap(err, 1)
}
rawSigWire := wire.NewWriteContext(signatureWriter)
rawSigWire.WriteMagic(pwr.SignatureMagic)
rawSigWire.WriteMessage(&pwr.SignatureHeader{
Compression: &compression,
})
sigWire, err := pwr.CompressWire(rawSigWire, &compression)
if err != nil {
return errors.Wrap(err, 1)
}
sigWire.WriteMessage(container)
comm.StartProgress()
err = pwr.ComputeSignatureToWriter(container, pool, comm.NewStateConsumer(), func(hash wsync.BlockHash) error {
return sigWire.WriteMessage(&pwr.BlockHash{
WeakHash: hash.WeakHash,
StrongHash: hash.StrongHash,
})
})
comm.EndProgress()
if err != nil {
return errors.Wrap(err, 1)
}
err = sigWire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
prettySize := humanize.IBytes(uint64(container.Size))
perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond)
return nil
}
// CompressWire wraps a wire.WriteContext into a compressor, according to given settings,
// so that any messages written through the returned WriteContext will first be compressed.
func CompressWire(ctx *wire.WriteContext, compression *CompressionSettings) (*wire.WriteContext, error) {
if compression == nil {
return nil, errors.Wrap(fmt.Errorf("no compression specified"), 1)
}
if compression.Algorithm == CompressionAlgorithm_NONE {
return ctx, nil
}
compressor := compressors[compression.Algorithm]
if compressor == nil {
return nil, errors.Wrap(fmt.Errorf("no compressor registered for %s", compression.Algorithm.String()), 1)
}
compressedWriter, err := compressor.Apply(ctx.Writer(), compression.Quality)
if err != nil {
return nil, errors.Wrap(err, 1)
}
return wire.NewWriteContext(compressedWriter), nil
}
func Test_Compression(t *testing.T) {
fc := &fakeCompressor{}
RegisterCompressor(CompressionAlgorithm_GZIP, fc)
fd := &fakeDecompressor{}
RegisterDecompressor(CompressionAlgorithm_GZIP, fd)
assert.EqualValues(t, false, fc.called)
buf := new(bytes.Buffer)
wc := wire.NewWriteContext(buf)
_, err := CompressWire(wc, &CompressionSettings{
Algorithm: CompressionAlgorithm_BROTLI,
Quality: 3,
})
assert.NotNil(t, err)
cwc, err := CompressWire(wc, &CompressionSettings{
Algorithm: CompressionAlgorithm_GZIP,
Quality: 3,
})
assert.NoError(t, err)
assert.EqualValues(t, true, fc.called)
assert.EqualValues(t, 3, fc.quality)
assert.NoError(t, cwc.WriteMessage(&SyncHeader{
FileIndex: 672,
}))
rc := wire.NewReadContext(bytes.NewReader(buf.Bytes()))
sh := &SyncHeader{}
assert.NoError(t, rc.ReadMessage(sh))
assert.EqualValues(t, 672, sh.FileIndex)
assert.NotNil(t, rc.ReadMessage(sh))
}
// Do only create a file at WoundsPath when it receives the first wound.
// If no wounds are ever received, Do will effectively be a no-op.
func (ww *WoundsWriter) Do(container *tlc.Container, wounds chan *Wound) error {
var fw *os.File
var wc *wire.WriteContext
defer func() {
if wc != nil {
wc.Close()
}
if fw != nil {
fw.Close()
}
}()
writeWound := func(wound *Wound) error {
ww.totalCorrupted += wound.Size()
if wc == nil {
var err error
fw, err = os.Create(ww.WoundsPath)
if err != nil {
return errors.Wrap(err, 1)
}
wc = wire.NewWriteContext(fw)
if err != nil {
return errors.Wrap(err, 1)
}
err = wc.WriteMagic(WoundsMagic)
if err != nil {
return errors.Wrap(err, 1)
}
err = wc.WriteMessage(&WoundsHeader{})
if err != nil {
return errors.Wrap(err, 1)
}
err = wc.WriteMessage(container)
if err != nil {
return errors.Wrap(err, 1)
}
}
err := wc.WriteMessage(wound)
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}
for wound := range wounds {
if wound.Healthy() {
continue
}
ww.hasWounds = true
err := writeWound(wound)
if err != nil {
return err
}
}
return nil
}
func doCmdBsdiff(target string, source string, patch string, concurrency int, measureOverhead bool) error {
targetReader, err := os.Open(target)
if err != nil {
return err
}
defer targetReader.Close()
targetStats, err := targetReader.Stat()
if err != nil {
return err
}
sourceReader, err := os.Open(source)
if err != nil {
return err
}
defer sourceReader.Close()
sourceStats, err := sourceReader.Stat()
if err != nil {
return err
}
comm.Opf("Diffing %s (%s) and %s (%s)...", target, humanize.IBytes(uint64(targetStats.Size())), source,
humanize.IBytes(uint64(sourceStats.Size())))
patchWriter, err := os.Create(patch)
if err != nil {
return err
}
wctx := wire.NewWriteContext(patchWriter)
err = wctx.WriteMagic(pwr.PatchMagic)
if err != nil {
return err
}
compression := butlerCompressionSettings()
err = wctx.WriteMessage(&pwr.PatchHeader{
Compression: &compression,
})
if err != nil {
return err
}
wctx, err = pwr.CompressWire(wctx, &compression)
if err != nil {
return err
}
targetContainer := &tlc.Container{}
targetContainer.Files = []*tlc.File{
&tlc.File{
Path: target,
Size: targetStats.Size(),
},
}
err = wctx.WriteMessage(targetContainer)
if err != nil {
return err
}
sourceContainer := &tlc.Container{}
sourceContainer.Files = []*tlc.File{
&tlc.File{
Path: source,
Size: sourceStats.Size(),
},
}
err = wctx.WriteMessage(sourceContainer)
if err != nil {
return err
}
err = wctx.WriteMessage(&pwr.SyncHeader{
FileIndex: 0,
})
if err != nil {
return err
}
err = wctx.WriteMessage(&pwr.SyncOp{
Type: pwr.SyncOp_BSDIFF,
FileIndex: 0,
})
if err != nil {
return err
}
startTime := time.Now()
comm.StartProgress()
dc := bsdiff.DiffContext{
MeasureMem: *appArgs.memstats,
MeasureParallelOverhead: measureOverhead,
SuffixSortConcurrency: concurrency,
}
err = dc.Do(targetReader, sourceReader, wctx.WriteMessage, comm.NewStateConsumer())
if err != nil {
return err
}
comm.EndProgress()
err = wctx.WriteMessage(&pwr.SyncOp{
Type: pwr.SyncOp_HEY_YOU_DID_IT,
})
if err != nil {
return err
}
err = wctx.Close()
if err != nil {
return err
}
patchStats, err := os.Lstat(patch)
if err != nil {
return err
}
duration := time.Since(startTime)
perSec := float64(sourceStats.Size()) / duration.Seconds()
relToNew := 100.0 * float64(patchStats.Size()) / float64(sourceStats.Size())
comm.Statf("Processed %s @ %s / s, total %s", humanize.IBytes(uint64(sourceStats.Size())), humanize.IBytes(uint64(perSec)), duration)
comm.Statf("Wrote %s patch (%.2f%% of total size) to %s", humanize.IBytes(uint64(patchStats.Size())), relToNew, patch)
return nil
}
// WriteManifest writes container info and block addresses in wharf's manifest format
// Does not close manifestWriter.
func WriteManifest(manifestWriter io.Writer, compression *pwr.CompressionSettings, container *tlc.Container, blockHashes *BlockHashMap) error {
rawWire := wire.NewWriteContext(manifestWriter)
err := rawWire.WriteMagic(pwr.ManifestMagic)
if err != nil {
return errors.Wrap(err, 1)
}
err = rawWire.WriteMessage(&pwr.ManifestHeader{
Compression: compression,
Algorithm: pwr.HashAlgorithm_SHAKE128_32,
})
if err != nil {
return errors.Wrap(err, 1)
}
wire, err := pwr.CompressWire(rawWire, compression)
if err != nil {
return errors.Wrap(err, 1)
}
err = wire.WriteMessage(container)
if err != nil {
return errors.Wrap(err, 1)
}
sh := &pwr.SyncHeader{}
mbh := &pwr.ManifestBlockHash{}
for fileIndex, f := range container.Files {
sh.Reset()
sh.FileIndex = int64(fileIndex)
err = wire.WriteMessage(sh)
if err != nil {
return errors.Wrap(err, 1)
}
numBlocks := ComputeNumBlocks(f.Size)
for blockIndex := int64(0); blockIndex < numBlocks; blockIndex++ {
loc := BlockLocation{FileIndex: int64(fileIndex), BlockIndex: blockIndex}
hash := blockHashes.Get(loc)
if hash == nil {
err = fmt.Errorf("missing BlockHash for block %+v", loc)
return errors.Wrap(err, 1)
}
mbh.Reset()
mbh.Hash = hash
err = wire.WriteMessage(mbh)
if err != nil {
return errors.Wrap(err, 1)
}
}
}
err = wire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}
// WritePatch outputs a pwr patch to patchWriter
func (dctx *DiffContext) WritePatch(patchWriter io.Writer, signatureWriter io.Writer) error {
if dctx.Compression == nil {
return errors.Wrap(fmt.Errorf("No compression settings specified, bailing out"), 1)
}
// signature header
rawSigWire := wire.NewWriteContext(signatureWriter)
err := rawSigWire.WriteMagic(SignatureMagic)
if err != nil {
return errors.Wrap(err, 1)
}
err = rawSigWire.WriteMessage(&SignatureHeader{
Compression: dctx.Compression,
})
if err != nil {
return errors.Wrap(err, 1)
}
sigWire, err := CompressWire(rawSigWire, dctx.Compression)
if err != nil {
return errors.Wrap(err, 1)
}
err = sigWire.WriteMessage(dctx.SourceContainer)
if err != nil {
return errors.Wrap(err, 1)
}
// patch header
rawPatchWire := wire.NewWriteContext(patchWriter)
err = rawPatchWire.WriteMagic(PatchMagic)
if err != nil {
return errors.Wrap(err, 1)
}
header := &PatchHeader{
Compression: dctx.Compression,
}
err = rawPatchWire.WriteMessage(header)
if err != nil {
return errors.Wrap(err, 1)
}
patchWire, err := CompressWire(rawPatchWire, dctx.Compression)
if err != nil {
return errors.Wrap(err, 1)
}
err = patchWire.WriteMessage(dctx.TargetContainer)
if err != nil {
return errors.Wrap(err, 1)
}
err = patchWire.WriteMessage(dctx.SourceContainer)
if err != nil {
return errors.Wrap(err, 1)
}
sourceBytes := dctx.SourceContainer.Size
fileOffset := int64(0)
onSourceRead := func(count int64) {
dctx.Consumer.Progress(float64(fileOffset+count) / float64(sourceBytes))
}
sigWriter := makeSigWriter(sigWire)
opsWriter := makeOpsWriter(patchWire, dctx)
diffContext := mksync()
signContext := mksync()
blockLibrary := wsync.NewBlockLibrary(dctx.TargetSignature)
targetContainerPathToIndex := make(map[string]int64)
for index, f := range dctx.TargetContainer.Files {
targetContainerPathToIndex[f.Path] = int64(index)
}
// re-used messages
syncHeader := &SyncHeader{}
syncDelimiter := &SyncOp{
Type: SyncOp_HEY_YOU_DID_IT,
}
pool := dctx.Pool
defer func() {
if fErr := pool.Close(); fErr != nil && err == nil {
err = errors.Wrap(fErr, 1)
}
}()
for fileIndex, f := range dctx.SourceContainer.Files {
dctx.Consumer.ProgressLabel(f.Path)
dctx.Consumer.Debug(fmt.Sprintf("%s (%s)", f.Path, humanize.IBytes(uint64(f.Size))))
fileOffset = f.Offset
syncHeader.Reset()
syncHeader.FileIndex = int64(fileIndex)
err = patchWire.WriteMessage(syncHeader)
if err != nil {
return errors.Wrap(err, 1)
}
var sourceReader io.Reader
sourceReader, err = pool.GetReader(int64(fileIndex))
if err != nil {
return errors.Wrap(err, 1)
}
// / differ
// source file +
// \ signer
diffReader, diffWriter := io.Pipe()
signReader, signWriter := io.Pipe()
done := make(chan bool)
errs := make(chan error)
var preferredFileIndex int64 = -1
if oldIndex, ok := targetContainerPathToIndex[f.Path]; ok {
preferredFileIndex = oldIndex
}
go diffFile(diffContext, dctx, blockLibrary, diffReader, opsWriter, preferredFileIndex, errs, done)
go signFile(signContext, fileIndex, signReader, sigWriter, errs, done)
go func() {
defer func() {
if dErr := diffWriter.Close(); dErr != nil {
errs <- errors.Wrap(dErr, 1)
}
}()
defer func() {
if sErr := signWriter.Close(); sErr != nil {
errs <- errors.Wrap(sErr, 1)
}
}()
mw := io.MultiWriter(diffWriter, signWriter)
sourceReadCounter := counter.NewReaderCallback(onSourceRead, sourceReader)
_, cErr := io.Copy(mw, sourceReadCounter)
if cErr != nil {
errs <- errors.Wrap(cErr, 1)
}
}()
// wait until all are done
// or an error occurs
for c := 0; c < 2; c++ {
select {
case wErr := <-errs:
return errors.Wrap(wErr, 1)
case <-done:
}
}
err = patchWire.WriteMessage(syncDelimiter)
if err != nil {
return errors.Wrap(err, 1)
}
}
err = patchWire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
err = sigWire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}