// ParseHeader is the first step of the genie's operation - it reads both
// containers, leaving the caller a chance to use them later, when parsing
// the contents
func (g *Genie) ParseHeader(patchReader io.Reader) error {
rawPatchWire := wire.NewReadContext(patchReader)
err := rawPatchWire.ExpectMagic(pwr.PatchMagic)
if err != nil {
return errors.Wrap(err, 1)
}
header := &pwr.PatchHeader{}
err = rawPatchWire.ReadMessage(header)
if err != nil {
return errors.Wrap(err, 1)
}
patchWire, err := pwr.DecompressWire(rawPatchWire, header.Compression)
if err != nil {
return errors.Wrap(err, 1)
}
g.PatchWire = patchWire
g.TargetContainer = &tlc.Container{}
err = patchWire.ReadMessage(g.TargetContainer)
if err != nil {
return errors.Wrap(err, 1)
}
g.SourceContainer = &tlc.Container{}
err = patchWire.ReadMessage(g.SourceContainer)
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}
// DecompressWire wraps a wire.ReadContext into a decompressor, according to the given settings,
// so that any messages read through the returned ReadContext will first be decompressed.
func DecompressWire(ctx *wire.ReadContext, compression *CompressionSettings) (*wire.ReadContext, error) {
if compression == nil {
return nil, errors.Wrap(fmt.Errorf("no compression specified"), 1)
}
if compression.Algorithm == CompressionAlgorithm_NONE {
return ctx, nil
}
decompressor := decompressors[compression.Algorithm]
if decompressor == nil {
return nil, errors.Wrap(fmt.Errorf("no compressor registered for %s", compression.Algorithm.String()), 1)
}
compressedReader, err := decompressor.Apply(ctx.Reader())
if err != nil {
return nil, errors.Wrap(err, 1)
}
return wire.NewReadContext(compressedReader), nil
}
func Test_Compression(t *testing.T) {
fc := &fakeCompressor{}
RegisterCompressor(CompressionAlgorithm_GZIP, fc)
fd := &fakeDecompressor{}
RegisterDecompressor(CompressionAlgorithm_GZIP, fd)
assert.EqualValues(t, false, fc.called)
buf := new(bytes.Buffer)
wc := wire.NewWriteContext(buf)
_, err := CompressWire(wc, &CompressionSettings{
Algorithm: CompressionAlgorithm_BROTLI,
Quality: 3,
})
assert.NotNil(t, err)
cwc, err := CompressWire(wc, &CompressionSettings{
Algorithm: CompressionAlgorithm_GZIP,
Quality: 3,
})
assert.NoError(t, err)
assert.EqualValues(t, true, fc.called)
assert.EqualValues(t, 3, fc.quality)
assert.NoError(t, cwc.WriteMessage(&SyncHeader{
FileIndex: 672,
}))
rc := wire.NewReadContext(bytes.NewReader(buf.Bytes()))
sh := &SyncHeader{}
assert.NoError(t, rc.ReadMessage(sh))
assert.EqualValues(t, 672, sh.FileIndex)
assert.NotNil(t, rc.ReadMessage(sh))
}
func doProbe(patch string) error {
patchReader, err := eos.Open(patch)
if err != nil {
return err
}
defer patchReader.Close()
stats, err := patchReader.Stat()
if err != nil {
return err
}
comm.Statf("patch: %s", humanize.IBytes(uint64(stats.Size())))
rctx := wire.NewReadContext(patchReader)
err = rctx.ExpectMagic(pwr.PatchMagic)
if err != nil {
return err
}
header := &pwr.PatchHeader{}
err = rctx.ReadMessage(header)
if err != nil {
return err
}
rctx, err = pwr.DecompressWire(rctx, header.Compression)
if err != nil {
return err
}
target := &tlc.Container{}
err = rctx.ReadMessage(target)
if err != nil {
return err
}
source := &tlc.Container{}
err = rctx.ReadMessage(source)
if err != nil {
return err
}
comm.Statf("target: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats())
comm.Statf("source: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats())
var patchStats []patchStat
sh := &pwr.SyncHeader{}
rop := &pwr.SyncOp{}
for fileIndex, f := range source.Files {
stat := patchStat{
fileIndex: int64(fileIndex),
freshData: f.Size,
}
sh.Reset()
err = rctx.ReadMessage(sh)
if err != nil {
return err
}
if sh.FileIndex != int64(fileIndex) {
return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex)
}
readingOps := true
var pos int64
for readingOps {
rop.Reset()
err = rctx.ReadMessage(rop)
if err != nil {
return err
}
switch rop.Type {
case pwr.SyncOp_BLOCK_RANGE:
fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize
lastIndex := rop.BlockIndex + (rop.BlockSpan - 1)
lastSize := pwr.ComputeBlockSize(f.Size, lastIndex)
totalSize := (fixedSize + lastSize)
stat.freshData -= totalSize
pos += totalSize
case pwr.SyncOp_DATA:
totalSize := int64(len(rop.Data))
if *appArgs.verbose {
comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)),
pos, pos+totalSize)
}
pos += totalSize
case pwr.SyncOp_HEY_YOU_DID_IT:
readingOps = false
}
}
patchStats = append(patchStats, stat)
}
sort.Sort(byDecreasingFreshData(patchStats))
var totalFresh int64
for _, stat := range patchStats {
totalFresh += stat.freshData
}
var eightyFresh = int64(0.8 * float64(totalFresh))
var printedFresh int64
comm.Opf("80%% of fresh data is in the following files:")
for _, stat := range patchStats {
f := source.Files[stat.fileIndex]
comm.Logf("%s in %s (%.2f%% changed)",
humanize.IBytes(uint64(stat.freshData)),
f.Path,
float64(stat.freshData)/float64(f.Size)*100.0)
printedFresh += stat.freshData
if printedFresh >= eightyFresh {
break
}
}
return nil
}
func doHeal(dir string, woundsPath string, spec string) error {
reader, err := os.Open(woundsPath)
if err != nil {
return err
}
defer reader.Close()
healer, err := pwr.NewHealer(spec, dir)
if err != nil {
return err
}
consumer := comm.NewStateConsumer()
healer.SetConsumer(consumer)
rctx := wire.NewReadContext(reader)
err = rctx.ExpectMagic(pwr.WoundsMagic)
if err != nil {
return err
}
wh := &pwr.WoundsHeader{}
err = rctx.ReadMessage(wh)
if err != nil {
return err
}
container := &tlc.Container{}
err = rctx.ReadMessage(container)
if err != nil {
return err
}
wounds := make(chan *pwr.Wound)
errs := make(chan error)
comm.StartProgress()
go func() {
errs <- healer.Do(container, wounds)
}()
wound := &pwr.Wound{}
for {
wound.Reset()
err = rctx.ReadMessage(wound)
if err != nil {
if err == io.EOF {
// all good
break
}
}
select {
case wounds <- wound:
// all good
case healErr := <-errs:
return healErr
}
}
close(wounds)
healErr := <-errs
comm.EndProgress()
if healErr != nil {
return healErr
}
comm.Opf("All healed!")
return nil
}
// ReadSignature reads the hashes from all files of a given container, from a
// wharf signature file.
func ReadSignature(signatureReader io.Reader) (*SignatureInfo, error) {
rawSigWire := wire.NewReadContext(signatureReader)
err := rawSigWire.ExpectMagic(SignatureMagic)
if err != nil {
return nil, errors.Wrap(err, 1)
}
header := &SignatureHeader{}
err = rawSigWire.ReadMessage(header)
if err != nil {
return nil, errors.Wrap(err, 1)
}
sigWire, err := DecompressWire(rawSigWire, header.Compression)
if err != nil {
return nil, errors.Wrap(err, 1)
}
container := &tlc.Container{}
err = sigWire.ReadMessage(container)
if err != nil {
if errors.Is(err, io.EOF) {
// ok
} else {
return nil, errors.Wrap(err, 1)
}
}
var hashes []wsync.BlockHash
hash := &BlockHash{}
for fileIndex, f := range container.Files {
numBlocks := ComputeNumBlocks(f.Size)
if numBlocks == 0 {
hash.Reset()
err = sigWire.ReadMessage(hash)
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, errors.Wrap(err, 1)
}
// empty files have a 0-length shortblock for historical reasons.
blockHash := wsync.BlockHash{
FileIndex: int64(fileIndex),
BlockIndex: 0,
WeakHash: hash.WeakHash,
StrongHash: hash.StrongHash,
ShortSize: 0,
}
hashes = append(hashes, blockHash)
}
for blockIndex := int64(0); blockIndex < numBlocks; blockIndex++ {
hash.Reset()
err = sigWire.ReadMessage(hash)
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, errors.Wrap(err, 1)
}
// full blocks have a shortSize of 0, for more compact storage
shortSize := int32(0)
if (blockIndex+1)*BlockSize > f.Size {
shortSize = int32(f.Size % BlockSize)
}
blockHash := wsync.BlockHash{
FileIndex: int64(fileIndex),
BlockIndex: blockIndex,
WeakHash: hash.WeakHash,
StrongHash: hash.StrongHash,
ShortSize: shortSize,
}
hashes = append(hashes, blockHash)
}
}
signature := &SignatureInfo{
Container: container,
Hashes: hashes,
}
return signature, nil
}
func doBspatch(patch string, target string, output string) error {
targetReader, err := os.Open(target)
if err != nil {
return err
}
defer targetReader.Close()
patchReader, err := os.Open(patch)
if err != nil {
return err
}
defer patchReader.Close()
err = os.MkdirAll(filepath.Dir(output), 0755)
if err != nil {
return err
}
outputWriter, err := os.Create(output)
if err != nil {
return err
}
defer outputWriter.Close()
rctx := wire.NewReadContext(patchReader)
err = rctx.ExpectMagic(pwr.PatchMagic)
if err != nil {
return err
}
ph := &pwr.PatchHeader{}
err = rctx.ReadMessage(ph)
if err != nil {
return err
}
compression := ph.GetCompression()
rctx, err = pwr.DecompressWire(rctx, compression)
if err != nil {
return err
}
targetContainer := &tlc.Container{}
err = rctx.ReadMessage(targetContainer)
if err != nil {
return err
}
sourceContainer := &tlc.Container{}
err = rctx.ReadMessage(sourceContainer)
if err != nil {
return err
}
if len(targetContainer.Files) != 1 {
return fmt.Errorf("expected only one file in target container")
}
if len(sourceContainer.Files) != 1 {
return fmt.Errorf("expected only one file in source container")
}
sh := &pwr.SyncHeader{}
err = rctx.ReadMessage(sh)
if err != nil {
return err
}
if sh.FileIndex != 0 {
return fmt.Errorf("wrong sync header")
}
op := &pwr.SyncOp{}
err = rctx.ReadMessage(op)
if err != nil {
return err
}
if op.Type != pwr.SyncOp_BSDIFF {
return fmt.Errorf("expected bsdiff syncop")
}
if op.FileIndex != 0 {
return fmt.Errorf("expected bsdiff syncop to operate on only file")
}
outputSize := sourceContainer.Files[op.FileIndex].Size
err = bsdiff.Patch(targetReader, outputWriter, outputSize, rctx.ReadMessage)
if err != nil {
return err
}
return nil
}
// ApplyPatch reads a patch, parses it, and generates the new file tree
func (actx *ApplyContext) ApplyPatch(patchReader io.Reader) error {
actx.actualOutputPath = actx.OutputPath
if actx.OutputPool == nil {
if actx.InPlace {
// applying in-place is a bit tricky: we can't overwrite files in the
// target directory (old) while we're reading the patch otherwise
// we might be copying new bytes instead of old bytes into later files
// so, we rebuild 'touched' files in a staging area
stagePath := actx.actualOutputPath + "-stage"
err := os.MkdirAll(stagePath, os.FileMode(0755))
if err != nil {
return errors.Wrap(err, 1)
}
defer os.RemoveAll(stagePath)
actx.OutputPath = stagePath
} else {
os.MkdirAll(actx.OutputPath, os.FileMode(0755))
}
} else {
if actx.actualOutputPath != "" {
return fmt.Errorf("cannot specify both OutputPath and OutputPool")
}
}
rawPatchWire := wire.NewReadContext(patchReader)
err := rawPatchWire.ExpectMagic(PatchMagic)
if err != nil {
return errors.Wrap(err, 1)
}
header := &PatchHeader{}
err = rawPatchWire.ReadMessage(header)
if err != nil {
return errors.Wrap(err, 1)
}
patchWire, err := DecompressWire(rawPatchWire, header.Compression)
if err != nil {
return errors.Wrap(err, 1)
}
targetContainer := &tlc.Container{}
err = patchWire.ReadMessage(targetContainer)
if err != nil {
return errors.Wrap(err, 1)
}
actx.TargetContainer = targetContainer
sourceContainer := &tlc.Container{}
err = patchWire.ReadMessage(sourceContainer)
if err != nil {
return errors.Wrap(err, 1)
}
actx.SourceContainer = sourceContainer
if actx.VetApply != nil {
err = actx.VetApply(actx)
if err != nil {
return errors.Wrap(err, 1)
}
}
var ghosts []Ghost
// when not working with a custom output pool
if actx.OutputPool == nil {
if actx.InPlace {
// when working in-place, we have to keep track of which files were deleted
// from one version to the other, so that we too may delete them in the end.
ghosts = detectGhosts(actx.SourceContainer, actx.TargetContainer)
} else {
// when rebuilding in a fresh directory, there's no need to worry about
// deleted files, because they won't even exist in the first place.
err = sourceContainer.Prepare(actx.OutputPath)
if err != nil {
return errors.Wrap(err, 1)
}
}
}
err = actx.patchAll(patchWire, actx.Signature)
if err != nil {
return errors.Wrap(err, 1)
}
if actx.InPlace {
err = actx.ensureDirsAndSymlinks(actx.actualOutputPath)
if err != nil {
return errors.Wrap(err, 1)
}
actx.Stats.StageSize, err = actx.mergeFolders(actx.actualOutputPath, actx.OutputPath)
if err != nil {
return errors.Wrap(err, 1)
}
err = actx.deleteGhosts(actx.actualOutputPath, ghosts)
if err != nil {
return errors.Wrap(err, 1)
}
actx.OutputPath = actx.actualOutputPath
}
return nil
}
// ReadManifest reads container info and block addresses from a wharf manifest file.
// Does not close manifestReader.
func ReadManifest(manifestReader io.Reader) (*tlc.Container, *BlockHashMap, error) {
container := &tlc.Container{}
blockHashes := NewBlockHashMap()
rawWire := wire.NewReadContext(manifestReader)
err := rawWire.ExpectMagic(pwr.ManifestMagic)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
mh := &pwr.ManifestHeader{}
err = rawWire.ReadMessage(mh)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
if mh.Algorithm != pwr.HashAlgorithm_SHAKE128_32 {
err = fmt.Errorf("Manifest has unsupported hash algorithm %d, expected %d", mh.Algorithm, pwr.HashAlgorithm_SHAKE128_32)
return nil, nil, errors.Wrap(err, 1)
}
wire, err := pwr.DecompressWire(rawWire, mh.GetCompression())
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
err = wire.ReadMessage(container)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
sh := &pwr.SyncHeader{}
mbh := &pwr.ManifestBlockHash{}
for fileIndex, f := range container.Files {
sh.Reset()
err = wire.ReadMessage(sh)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
if int64(fileIndex) != sh.FileIndex {
err = fmt.Errorf("manifest format error: expected file %d, got %d", fileIndex, sh.FileIndex)
return nil, nil, errors.Wrap(err, 1)
}
numBlocks := ComputeNumBlocks(f.Size)
for blockIndex := int64(0); blockIndex < numBlocks; blockIndex++ {
mbh.Reset()
err = wire.ReadMessage(mbh)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
loc := BlockLocation{FileIndex: int64(fileIndex), BlockIndex: blockIndex}
blockHashes.Set(loc, append([]byte{}, mbh.Hash...))
}
}
return container, blockHashes, nil
}
func file(path string) {
reader, err := eos.Open(path)
must(err)
defer reader.Close()
stats, err := reader.Stat()
if os.IsNotExist(err) {
comm.Dief("%s: no such file or directory", path)
}
must(err)
if stats.IsDir() {
comm.Logf("%s: directory", path)
return
}
if stats.Size() == 0 {
comm.Logf("%s: empty file. peaceful.", path)
return
}
prettySize := humanize.IBytes(uint64(stats.Size()))
var magic int32
must(binary.Read(reader, wire.Endianness, &magic))
switch magic {
case pwr.PatchMagic:
{
ph := &pwr.PatchHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(ph))
rctx, err = pwr.DecompressWire(rctx, ph.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container)) // target container
container.Reset()
must(rctx.ReadMessage(container)) // source container
comm.Logf("%s: %s wharf patch file (%s) with %s", path, prettySize, ph.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/patch",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.SignatureMagic:
{
sh := &pwr.SignatureHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(sh))
rctx, err = pwr.DecompressWire(rctx, sh.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
comm.Logf("%s: %s wharf signature file (%s) with %s", path, prettySize, sh.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/signature",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.ManifestMagic:
{
mh := &pwr.ManifestHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(mh))
rctx, err = pwr.DecompressWire(rctx, mh.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
comm.Logf("%s: %s wharf manifest file (%s) with %s", path, prettySize, mh.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/manifest",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.WoundsMagic:
{
wh := &pwr.WoundsHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(wh))
container := &tlc.Container{}
must(rctx.ReadMessage(container))
files := make(map[int64]bool)
totalWounds := int64(0)
for {
wound := &pwr.Wound{}
err = rctx.ReadMessage(wound)
if err != nil {
if errors.Is(err, io.EOF) {
break
} else {
must(err)
}
}
if wound.Kind == pwr.WoundKind_FILE {
totalWounds += (wound.End - wound.Start)
files[wound.Index] = true
}
}
comm.Logf("%s: %s wharf wounds file with %s, %s wounds in %d files", path, prettySize, container.Stats(),
humanize.IBytes(uint64(totalWounds)), len(files))
comm.Result(ContainerResult{
Type: "wharf/wounds",
})
}
default:
_, err := reader.Seek(0, os.SEEK_SET)
must(err)
wasZip := func() bool {
zr, err := zip.NewReader(reader, stats.Size())
if err != nil {
if err != zip.ErrFormat {
must(err)
}
return false
}
container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
must(err)
comm.Logf("%s: %s zip file with %s", path, prettySize, container.Stats())
comm.Result(ContainerResult{
Type: "zip",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
return true
}()
if !wasZip {
comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
}
}
}
func ls(path string) {
reader, err := eos.Open(path)
must(err)
defer reader.Close()
stats, err := reader.Stat()
if os.IsNotExist(err) {
comm.Dief("%s: no such file or directory", path)
}
must(err)
if stats.IsDir() {
comm.Logf("%s: directory", path)
return
}
if stats.Size() == 0 {
comm.Logf("%s: empty file. peaceful.", path)
return
}
log := func(line string) {
comm.Logf(line)
}
var magic int32
must(binary.Read(reader, wire.Endianness, &magic))
switch magic {
case pwr.PatchMagic:
{
h := &pwr.PatchHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
log("pre-patch container:")
container.Print(log)
container.Reset()
must(rctx.ReadMessage(container))
log("================================")
log("post-patch container:")
container.Print(log)
}
case pwr.SignatureMagic:
{
h := &pwr.SignatureHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
}
case pwr.ManifestMagic:
{
h := &pwr.ManifestHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
}
case pwr.WoundsMagic:
{
wh := &pwr.WoundsHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(wh))
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
for {
wound := &pwr.Wound{}
err = rctx.ReadMessage(wound)
if err != nil {
if errors.Is(err, io.EOF) {
break
} else {
must(err)
}
}
comm.Logf(wound.PrettyString(container))
}
}
default:
_, err := reader.Seek(0, os.SEEK_SET)
must(err)
wasZip := func() bool {
zr, err := zip.NewReader(reader, stats.Size())
if err != nil {
if err != zip.ErrFormat {
must(err)
}
return false
}
container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
must(err)
container.Print(log)
return true
}()
if !wasZip {
comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
}
}
}