// ParseHeader is the first step of the genie's operation - it reads both
// containers, leaving the caller a chance to use them later, when parsing
// the contents
func (g *Genie) ParseHeader(patchReader io.Reader) error {
rawPatchWire := wire.NewReadContext(patchReader)
err := rawPatchWire.ExpectMagic(pwr.PatchMagic)
if err != nil {
return errors.Wrap(err, 1)
}
header := &pwr.PatchHeader{}
err = rawPatchWire.ReadMessage(header)
if err != nil {
return errors.Wrap(err, 1)
}
patchWire, err := pwr.DecompressWire(rawPatchWire, header.Compression)
if err != nil {
return errors.Wrap(err, 1)
}
g.PatchWire = patchWire
g.TargetContainer = &tlc.Container{}
err = patchWire.ReadMessage(g.TargetContainer)
if err != nil {
return errors.Wrap(err, 1)
}
g.SourceContainer = &tlc.Container{}
err = patchWire.ReadMessage(g.SourceContainer)
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}
func doProbe(patch string) error {
patchReader, err := eos.Open(patch)
if err != nil {
return err
}
defer patchReader.Close()
stats, err := patchReader.Stat()
if err != nil {
return err
}
comm.Statf("patch: %s", humanize.IBytes(uint64(stats.Size())))
rctx := wire.NewReadContext(patchReader)
err = rctx.ExpectMagic(pwr.PatchMagic)
if err != nil {
return err
}
header := &pwr.PatchHeader{}
err = rctx.ReadMessage(header)
if err != nil {
return err
}
rctx, err = pwr.DecompressWire(rctx, header.Compression)
if err != nil {
return err
}
target := &tlc.Container{}
err = rctx.ReadMessage(target)
if err != nil {
return err
}
source := &tlc.Container{}
err = rctx.ReadMessage(source)
if err != nil {
return err
}
comm.Statf("target: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats())
comm.Statf("source: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats())
var patchStats []patchStat
sh := &pwr.SyncHeader{}
rop := &pwr.SyncOp{}
for fileIndex, f := range source.Files {
stat := patchStat{
fileIndex: int64(fileIndex),
freshData: f.Size,
}
sh.Reset()
err = rctx.ReadMessage(sh)
if err != nil {
return err
}
if sh.FileIndex != int64(fileIndex) {
return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex)
}
readingOps := true
var pos int64
for readingOps {
rop.Reset()
err = rctx.ReadMessage(rop)
if err != nil {
return err
}
switch rop.Type {
case pwr.SyncOp_BLOCK_RANGE:
fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize
lastIndex := rop.BlockIndex + (rop.BlockSpan - 1)
lastSize := pwr.ComputeBlockSize(f.Size, lastIndex)
totalSize := (fixedSize + lastSize)
stat.freshData -= totalSize
pos += totalSize
case pwr.SyncOp_DATA:
totalSize := int64(len(rop.Data))
if *appArgs.verbose {
comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)),
pos, pos+totalSize)
}
pos += totalSize
case pwr.SyncOp_HEY_YOU_DID_IT:
readingOps = false
}
}
patchStats = append(patchStats, stat)
}
sort.Sort(byDecreasingFreshData(patchStats))
var totalFresh int64
for _, stat := range patchStats {
totalFresh += stat.freshData
}
var eightyFresh = int64(0.8 * float64(totalFresh))
var printedFresh int64
comm.Opf("80%% of fresh data is in the following files:")
for _, stat := range patchStats {
f := source.Files[stat.fileIndex]
comm.Logf("%s in %s (%.2f%% changed)",
humanize.IBytes(uint64(stat.freshData)),
f.Path,
float64(stat.freshData)/float64(f.Size)*100.0)
printedFresh += stat.freshData
if printedFresh >= eightyFresh {
break
}
}
return nil
}
func doBspatch(patch string, target string, output string) error {
targetReader, err := os.Open(target)
if err != nil {
return err
}
defer targetReader.Close()
patchReader, err := os.Open(patch)
if err != nil {
return err
}
defer patchReader.Close()
err = os.MkdirAll(filepath.Dir(output), 0755)
if err != nil {
return err
}
outputWriter, err := os.Create(output)
if err != nil {
return err
}
defer outputWriter.Close()
rctx := wire.NewReadContext(patchReader)
err = rctx.ExpectMagic(pwr.PatchMagic)
if err != nil {
return err
}
ph := &pwr.PatchHeader{}
err = rctx.ReadMessage(ph)
if err != nil {
return err
}
compression := ph.GetCompression()
rctx, err = pwr.DecompressWire(rctx, compression)
if err != nil {
return err
}
targetContainer := &tlc.Container{}
err = rctx.ReadMessage(targetContainer)
if err != nil {
return err
}
sourceContainer := &tlc.Container{}
err = rctx.ReadMessage(sourceContainer)
if err != nil {
return err
}
if len(targetContainer.Files) != 1 {
return fmt.Errorf("expected only one file in target container")
}
if len(sourceContainer.Files) != 1 {
return fmt.Errorf("expected only one file in source container")
}
sh := &pwr.SyncHeader{}
err = rctx.ReadMessage(sh)
if err != nil {
return err
}
if sh.FileIndex != 0 {
return fmt.Errorf("wrong sync header")
}
op := &pwr.SyncOp{}
err = rctx.ReadMessage(op)
if err != nil {
return err
}
if op.Type != pwr.SyncOp_BSDIFF {
return fmt.Errorf("expected bsdiff syncop")
}
if op.FileIndex != 0 {
return fmt.Errorf("expected bsdiff syncop to operate on only file")
}
outputSize := sourceContainer.Files[op.FileIndex].Size
err = bsdiff.Patch(targetReader, outputWriter, outputSize, rctx.ReadMessage)
if err != nil {
return err
}
return nil
}
// ReadManifest reads container info and block addresses from a wharf manifest file.
// Does not close manifestReader.
func ReadManifest(manifestReader io.Reader) (*tlc.Container, *BlockHashMap, error) {
container := &tlc.Container{}
blockHashes := NewBlockHashMap()
rawWire := wire.NewReadContext(manifestReader)
err := rawWire.ExpectMagic(pwr.ManifestMagic)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
mh := &pwr.ManifestHeader{}
err = rawWire.ReadMessage(mh)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
if mh.Algorithm != pwr.HashAlgorithm_SHAKE128_32 {
err = fmt.Errorf("Manifest has unsupported hash algorithm %d, expected %d", mh.Algorithm, pwr.HashAlgorithm_SHAKE128_32)
return nil, nil, errors.Wrap(err, 1)
}
wire, err := pwr.DecompressWire(rawWire, mh.GetCompression())
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
err = wire.ReadMessage(container)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
sh := &pwr.SyncHeader{}
mbh := &pwr.ManifestBlockHash{}
for fileIndex, f := range container.Files {
sh.Reset()
err = wire.ReadMessage(sh)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
if int64(fileIndex) != sh.FileIndex {
err = fmt.Errorf("manifest format error: expected file %d, got %d", fileIndex, sh.FileIndex)
return nil, nil, errors.Wrap(err, 1)
}
numBlocks := ComputeNumBlocks(f.Size)
for blockIndex := int64(0); blockIndex < numBlocks; blockIndex++ {
mbh.Reset()
err = wire.ReadMessage(mbh)
if err != nil {
return nil, nil, errors.Wrap(err, 1)
}
loc := BlockLocation{FileIndex: int64(fileIndex), BlockIndex: blockIndex}
blockHashes.Set(loc, append([]byte{}, mbh.Hash...))
}
}
return container, blockHashes, nil
}
func file(path string) {
reader, err := eos.Open(path)
must(err)
defer reader.Close()
stats, err := reader.Stat()
if os.IsNotExist(err) {
comm.Dief("%s: no such file or directory", path)
}
must(err)
if stats.IsDir() {
comm.Logf("%s: directory", path)
return
}
if stats.Size() == 0 {
comm.Logf("%s: empty file. peaceful.", path)
return
}
prettySize := humanize.IBytes(uint64(stats.Size()))
var magic int32
must(binary.Read(reader, wire.Endianness, &magic))
switch magic {
case pwr.PatchMagic:
{
ph := &pwr.PatchHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(ph))
rctx, err = pwr.DecompressWire(rctx, ph.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container)) // target container
container.Reset()
must(rctx.ReadMessage(container)) // source container
comm.Logf("%s: %s wharf patch file (%s) with %s", path, prettySize, ph.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/patch",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.SignatureMagic:
{
sh := &pwr.SignatureHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(sh))
rctx, err = pwr.DecompressWire(rctx, sh.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
comm.Logf("%s: %s wharf signature file (%s) with %s", path, prettySize, sh.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/signature",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.ManifestMagic:
{
mh := &pwr.ManifestHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(mh))
rctx, err = pwr.DecompressWire(rctx, mh.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
comm.Logf("%s: %s wharf manifest file (%s) with %s", path, prettySize, mh.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/manifest",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.WoundsMagic:
{
wh := &pwr.WoundsHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(wh))
container := &tlc.Container{}
must(rctx.ReadMessage(container))
files := make(map[int64]bool)
totalWounds := int64(0)
for {
wound := &pwr.Wound{}
err = rctx.ReadMessage(wound)
if err != nil {
if errors.Is(err, io.EOF) {
break
} else {
must(err)
}
}
if wound.Kind == pwr.WoundKind_FILE {
totalWounds += (wound.End - wound.Start)
files[wound.Index] = true
}
}
comm.Logf("%s: %s wharf wounds file with %s, %s wounds in %d files", path, prettySize, container.Stats(),
humanize.IBytes(uint64(totalWounds)), len(files))
comm.Result(ContainerResult{
Type: "wharf/wounds",
})
}
default:
_, err := reader.Seek(0, os.SEEK_SET)
must(err)
wasZip := func() bool {
zr, err := zip.NewReader(reader, stats.Size())
if err != nil {
if err != zip.ErrFormat {
must(err)
}
return false
}
container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
must(err)
comm.Logf("%s: %s zip file with %s", path, prettySize, container.Stats())
comm.Result(ContainerResult{
Type: "zip",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
return true
}()
if !wasZip {
comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
}
}
}
func ls(path string) {
reader, err := eos.Open(path)
must(err)
defer reader.Close()
stats, err := reader.Stat()
if os.IsNotExist(err) {
comm.Dief("%s: no such file or directory", path)
}
must(err)
if stats.IsDir() {
comm.Logf("%s: directory", path)
return
}
if stats.Size() == 0 {
comm.Logf("%s: empty file. peaceful.", path)
return
}
log := func(line string) {
comm.Logf(line)
}
var magic int32
must(binary.Read(reader, wire.Endianness, &magic))
switch magic {
case pwr.PatchMagic:
{
h := &pwr.PatchHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
log("pre-patch container:")
container.Print(log)
container.Reset()
must(rctx.ReadMessage(container))
log("================================")
log("post-patch container:")
container.Print(log)
}
case pwr.SignatureMagic:
{
h := &pwr.SignatureHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
}
case pwr.ManifestMagic:
{
h := &pwr.ManifestHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
}
case pwr.WoundsMagic:
{
wh := &pwr.WoundsHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(wh))
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
for {
wound := &pwr.Wound{}
err = rctx.ReadMessage(wound)
if err != nil {
if errors.Is(err, io.EOF) {
break
} else {
must(err)
}
}
comm.Logf(wound.PrettyString(container))
}
}
default:
_, err := reader.Seek(0, os.SEEK_SET)
must(err)
wasZip := func() bool {
zr, err := zip.NewReader(reader, stats.Size())
if err != nil {
if err != zip.ErrFormat {
must(err)
}
return false
}
container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
must(err)
container.Print(log)
return true
}()
if !wasZip {
comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
}
}
}