func doSign(output string, signature string, compression pwr.CompressionSettings, fixPerms bool) error {
comm.Opf("Creating signature for %s", output)
startTime := time.Now()
container, err := tlc.WalkAny(output, filterPaths)
if err != nil {
return errors.Wrap(err, 1)
}
pool, err := pools.New(container, output)
if err != nil {
return errors.Wrap(err, 1)
}
if fixPerms {
container.FixPermissions(pool)
}
signatureWriter, err := os.Create(signature)
if err != nil {
return errors.Wrap(err, 1)
}
rawSigWire := wire.NewWriteContext(signatureWriter)
rawSigWire.WriteMagic(pwr.SignatureMagic)
rawSigWire.WriteMessage(&pwr.SignatureHeader{
Compression: &compression,
})
sigWire, err := pwr.CompressWire(rawSigWire, &compression)
if err != nil {
return errors.Wrap(err, 1)
}
sigWire.WriteMessage(container)
comm.StartProgress()
err = pwr.ComputeSignatureToWriter(container, pool, comm.NewStateConsumer(), func(hash wsync.BlockHash) error {
return sigWire.WriteMessage(&pwr.BlockHash{
WeakHash: hash.WeakHash,
StrongHash: hash.StrongHash,
})
})
comm.EndProgress()
if err != nil {
return errors.Wrap(err, 1)
}
err = sigWire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
prettySize := humanize.IBytes(uint64(container.Size))
perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond)
return nil
}
func doCmdBsdiff(target string, source string, patch string, concurrency int, measureOverhead bool) error {
targetReader, err := os.Open(target)
if err != nil {
return err
}
defer targetReader.Close()
targetStats, err := targetReader.Stat()
if err != nil {
return err
}
sourceReader, err := os.Open(source)
if err != nil {
return err
}
defer sourceReader.Close()
sourceStats, err := sourceReader.Stat()
if err != nil {
return err
}
comm.Opf("Diffing %s (%s) and %s (%s)...", target, humanize.IBytes(uint64(targetStats.Size())), source,
humanize.IBytes(uint64(sourceStats.Size())))
patchWriter, err := os.Create(patch)
if err != nil {
return err
}
wctx := wire.NewWriteContext(patchWriter)
err = wctx.WriteMagic(pwr.PatchMagic)
if err != nil {
return err
}
compression := butlerCompressionSettings()
err = wctx.WriteMessage(&pwr.PatchHeader{
Compression: &compression,
})
if err != nil {
return err
}
wctx, err = pwr.CompressWire(wctx, &compression)
if err != nil {
return err
}
targetContainer := &tlc.Container{}
targetContainer.Files = []*tlc.File{
&tlc.File{
Path: target,
Size: targetStats.Size(),
},
}
err = wctx.WriteMessage(targetContainer)
if err != nil {
return err
}
sourceContainer := &tlc.Container{}
sourceContainer.Files = []*tlc.File{
&tlc.File{
Path: source,
Size: sourceStats.Size(),
},
}
err = wctx.WriteMessage(sourceContainer)
if err != nil {
return err
}
err = wctx.WriteMessage(&pwr.SyncHeader{
FileIndex: 0,
})
if err != nil {
return err
}
err = wctx.WriteMessage(&pwr.SyncOp{
Type: pwr.SyncOp_BSDIFF,
FileIndex: 0,
})
if err != nil {
return err
}
startTime := time.Now()
comm.StartProgress()
dc := bsdiff.DiffContext{
MeasureMem: *appArgs.memstats,
MeasureParallelOverhead: measureOverhead,
SuffixSortConcurrency: concurrency,
}
err = dc.Do(targetReader, sourceReader, wctx.WriteMessage, comm.NewStateConsumer())
if err != nil {
return err
}
comm.EndProgress()
err = wctx.WriteMessage(&pwr.SyncOp{
Type: pwr.SyncOp_HEY_YOU_DID_IT,
})
if err != nil {
return err
}
err = wctx.Close()
if err != nil {
return err
}
patchStats, err := os.Lstat(patch)
if err != nil {
return err
}
duration := time.Since(startTime)
perSec := float64(sourceStats.Size()) / duration.Seconds()
relToNew := 100.0 * float64(patchStats.Size()) / float64(sourceStats.Size())
comm.Statf("Processed %s @ %s / s, total %s", humanize.IBytes(uint64(sourceStats.Size())), humanize.IBytes(uint64(perSec)), duration)
comm.Statf("Wrote %s patch (%.2f%% of total size) to %s", humanize.IBytes(uint64(patchStats.Size())), relToNew, patch)
return nil
}
// WriteManifest writes container info and block addresses in wharf's manifest format
// Does not close manifestWriter.
func WriteManifest(manifestWriter io.Writer, compression *pwr.CompressionSettings, container *tlc.Container, blockHashes *BlockHashMap) error {
rawWire := wire.NewWriteContext(manifestWriter)
err := rawWire.WriteMagic(pwr.ManifestMagic)
if err != nil {
return errors.Wrap(err, 1)
}
err = rawWire.WriteMessage(&pwr.ManifestHeader{
Compression: compression,
Algorithm: pwr.HashAlgorithm_SHAKE128_32,
})
if err != nil {
return errors.Wrap(err, 1)
}
wire, err := pwr.CompressWire(rawWire, compression)
if err != nil {
return errors.Wrap(err, 1)
}
err = wire.WriteMessage(container)
if err != nil {
return errors.Wrap(err, 1)
}
sh := &pwr.SyncHeader{}
mbh := &pwr.ManifestBlockHash{}
for fileIndex, f := range container.Files {
sh.Reset()
sh.FileIndex = int64(fileIndex)
err = wire.WriteMessage(sh)
if err != nil {
return errors.Wrap(err, 1)
}
numBlocks := ComputeNumBlocks(f.Size)
for blockIndex := int64(0); blockIndex < numBlocks; blockIndex++ {
loc := BlockLocation{FileIndex: int64(fileIndex), BlockIndex: blockIndex}
hash := blockHashes.Get(loc)
if hash == nil {
err = fmt.Errorf("missing BlockHash for block %+v", loc)
return errors.Wrap(err, 1)
}
mbh.Reset()
mbh.Hash = hash
err = wire.WriteMessage(mbh)
if err != nil {
return errors.Wrap(err, 1)
}
}
}
err = wire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}
