Exemplo n.º 1
0
// ParseHeader is the first step of the genie's operation - it reads both
// containers, leaving the caller a chance to use them later, when parsing
// the contents
func (g *Genie) ParseHeader(patchReader io.Reader) error {
	rawPatchWire := wire.NewReadContext(patchReader)
	err := rawPatchWire.ExpectMagic(pwr.PatchMagic)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	header := &pwr.PatchHeader{}
	err = rawPatchWire.ReadMessage(header)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	patchWire, err := pwr.DecompressWire(rawPatchWire, header.Compression)
	if err != nil {
		return errors.Wrap(err, 1)
	}
	g.PatchWire = patchWire

	g.TargetContainer = &tlc.Container{}
	err = patchWire.ReadMessage(g.TargetContainer)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	g.SourceContainer = &tlc.Container{}
	err = patchWire.ReadMessage(g.SourceContainer)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	return nil
}
Exemplo n.º 2
0
func doProbe(patch string) error {
	patchReader, err := eos.Open(patch)
	if err != nil {
		return err
	}

	defer patchReader.Close()

	stats, err := patchReader.Stat()
	if err != nil {
		return err
	}

	comm.Statf("patch:  %s", humanize.IBytes(uint64(stats.Size())))

	rctx := wire.NewReadContext(patchReader)
	err = rctx.ExpectMagic(pwr.PatchMagic)
	if err != nil {
		return err
	}

	header := &pwr.PatchHeader{}
	err = rctx.ReadMessage(header)
	if err != nil {
		return err
	}

	rctx, err = pwr.DecompressWire(rctx, header.Compression)
	if err != nil {
		return err
	}

	target := &tlc.Container{}
	err = rctx.ReadMessage(target)
	if err != nil {
		return err
	}

	source := &tlc.Container{}
	err = rctx.ReadMessage(source)
	if err != nil {
		return err
	}

	comm.Statf("target: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats())
	comm.Statf("source: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats())

	var patchStats []patchStat

	sh := &pwr.SyncHeader{}
	rop := &pwr.SyncOp{}

	for fileIndex, f := range source.Files {
		stat := patchStat{
			fileIndex: int64(fileIndex),
			freshData: f.Size,
		}

		sh.Reset()
		err = rctx.ReadMessage(sh)
		if err != nil {
			return err
		}

		if sh.FileIndex != int64(fileIndex) {
			return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex)
		}

		readingOps := true

		var pos int64

		for readingOps {
			rop.Reset()

			err = rctx.ReadMessage(rop)
			if err != nil {
				return err
			}

			switch rop.Type {
			case pwr.SyncOp_BLOCK_RANGE:
				fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize
				lastIndex := rop.BlockIndex + (rop.BlockSpan - 1)
				lastSize := pwr.ComputeBlockSize(f.Size, lastIndex)
				totalSize := (fixedSize + lastSize)
				stat.freshData -= totalSize
				pos += totalSize
			case pwr.SyncOp_DATA:
				totalSize := int64(len(rop.Data))
				if *appArgs.verbose {
					comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)),
						pos, pos+totalSize)
				}
				pos += totalSize
			case pwr.SyncOp_HEY_YOU_DID_IT:
				readingOps = false
			}
		}

		patchStats = append(patchStats, stat)
	}

	sort.Sort(byDecreasingFreshData(patchStats))

	var totalFresh int64
	for _, stat := range patchStats {
		totalFresh += stat.freshData
	}

	var eightyFresh = int64(0.8 * float64(totalFresh))
	var printedFresh int64

	comm.Opf("80%% of fresh data is in the following files:")

	for _, stat := range patchStats {
		f := source.Files[stat.fileIndex]
		comm.Logf("%s in %s (%.2f%% changed)",
			humanize.IBytes(uint64(stat.freshData)),
			f.Path,
			float64(stat.freshData)/float64(f.Size)*100.0)

		printedFresh += stat.freshData
		if printedFresh >= eightyFresh {
			break
		}
	}

	return nil
}
Exemplo n.º 3
0
func doBspatch(patch string, target string, output string) error {
	targetReader, err := os.Open(target)
	if err != nil {
		return err
	}

	defer targetReader.Close()

	patchReader, err := os.Open(patch)
	if err != nil {
		return err
	}

	defer patchReader.Close()

	err = os.MkdirAll(filepath.Dir(output), 0755)
	if err != nil {
		return err
	}

	outputWriter, err := os.Create(output)
	if err != nil {
		return err
	}

	defer outputWriter.Close()

	rctx := wire.NewReadContext(patchReader)

	err = rctx.ExpectMagic(pwr.PatchMagic)
	if err != nil {
		return err
	}

	ph := &pwr.PatchHeader{}

	err = rctx.ReadMessage(ph)
	if err != nil {
		return err
	}

	compression := ph.GetCompression()

	rctx, err = pwr.DecompressWire(rctx, compression)
	if err != nil {
		return err
	}

	targetContainer := &tlc.Container{}
	err = rctx.ReadMessage(targetContainer)
	if err != nil {
		return err
	}

	sourceContainer := &tlc.Container{}
	err = rctx.ReadMessage(sourceContainer)
	if err != nil {
		return err
	}

	if len(targetContainer.Files) != 1 {
		return fmt.Errorf("expected only one file in target container")
	}

	if len(sourceContainer.Files) != 1 {
		return fmt.Errorf("expected only one file in source container")
	}

	sh := &pwr.SyncHeader{}
	err = rctx.ReadMessage(sh)
	if err != nil {
		return err
	}

	if sh.FileIndex != 0 {
		return fmt.Errorf("wrong sync header")
	}

	op := &pwr.SyncOp{}
	err = rctx.ReadMessage(op)
	if err != nil {
		return err
	}

	if op.Type != pwr.SyncOp_BSDIFF {
		return fmt.Errorf("expected bsdiff syncop")
	}

	if op.FileIndex != 0 {
		return fmt.Errorf("expected bsdiff syncop to operate on only file")
	}

	outputSize := sourceContainer.Files[op.FileIndex].Size

	err = bsdiff.Patch(targetReader, outputWriter, outputSize, rctx.ReadMessage)
	if err != nil {
		return err
	}

	return nil
}
Exemplo n.º 4
0
// ReadManifest reads container info and block addresses from a wharf manifest file.
// Does not close manifestReader.
func ReadManifest(manifestReader io.Reader) (*tlc.Container, *BlockHashMap, error) {
	container := &tlc.Container{}
	blockHashes := NewBlockHashMap()

	rawWire := wire.NewReadContext(manifestReader)
	err := rawWire.ExpectMagic(pwr.ManifestMagic)
	if err != nil {
		return nil, nil, errors.Wrap(err, 1)
	}

	mh := &pwr.ManifestHeader{}
	err = rawWire.ReadMessage(mh)
	if err != nil {
		return nil, nil, errors.Wrap(err, 1)
	}

	if mh.Algorithm != pwr.HashAlgorithm_SHAKE128_32 {
		err = fmt.Errorf("Manifest has unsupported hash algorithm %d, expected %d", mh.Algorithm, pwr.HashAlgorithm_SHAKE128_32)
		return nil, nil, errors.Wrap(err, 1)
	}

	wire, err := pwr.DecompressWire(rawWire, mh.GetCompression())
	if err != nil {
		return nil, nil, errors.Wrap(err, 1)
	}

	err = wire.ReadMessage(container)
	if err != nil {
		return nil, nil, errors.Wrap(err, 1)
	}

	sh := &pwr.SyncHeader{}
	mbh := &pwr.ManifestBlockHash{}

	for fileIndex, f := range container.Files {
		sh.Reset()
		err = wire.ReadMessage(sh)
		if err != nil {
			return nil, nil, errors.Wrap(err, 1)
		}

		if int64(fileIndex) != sh.FileIndex {
			err = fmt.Errorf("manifest format error: expected file %d, got %d", fileIndex, sh.FileIndex)
			return nil, nil, errors.Wrap(err, 1)
		}

		numBlocks := ComputeNumBlocks(f.Size)
		for blockIndex := int64(0); blockIndex < numBlocks; blockIndex++ {
			mbh.Reset()
			err = wire.ReadMessage(mbh)
			if err != nil {
				return nil, nil, errors.Wrap(err, 1)
			}

			loc := BlockLocation{FileIndex: int64(fileIndex), BlockIndex: blockIndex}
			blockHashes.Set(loc, append([]byte{}, mbh.Hash...))
		}
	}

	return container, blockHashes, nil
}
Exemplo n.º 5
0
func file(path string) {
	reader, err := eos.Open(path)
	must(err)

	defer reader.Close()

	stats, err := reader.Stat()
	if os.IsNotExist(err) {
		comm.Dief("%s: no such file or directory", path)
	}
	must(err)

	if stats.IsDir() {
		comm.Logf("%s: directory", path)
		return
	}

	if stats.Size() == 0 {
		comm.Logf("%s: empty file. peaceful.", path)
		return
	}

	prettySize := humanize.IBytes(uint64(stats.Size()))

	var magic int32
	must(binary.Read(reader, wire.Endianness, &magic))

	switch magic {
	case pwr.PatchMagic:
		{
			ph := &pwr.PatchHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(ph))

			rctx, err = pwr.DecompressWire(rctx, ph.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container)) // target container
			container.Reset()
			must(rctx.ReadMessage(container)) // source container

			comm.Logf("%s: %s wharf patch file (%s) with %s", path, prettySize, ph.GetCompression().ToString(), container.Stats())
			comm.Result(ContainerResult{
				Type:             "wharf/patch",
				NumFiles:         len(container.Files),
				NumDirs:          len(container.Dirs),
				NumSymlinks:      len(container.Symlinks),
				UncompressedSize: container.Size,
			})
		}

	case pwr.SignatureMagic:
		{
			sh := &pwr.SignatureHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(sh))

			rctx, err = pwr.DecompressWire(rctx, sh.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))

			comm.Logf("%s: %s wharf signature file (%s) with %s", path, prettySize, sh.GetCompression().ToString(), container.Stats())
			comm.Result(ContainerResult{
				Type:             "wharf/signature",
				NumFiles:         len(container.Files),
				NumDirs:          len(container.Dirs),
				NumSymlinks:      len(container.Symlinks),
				UncompressedSize: container.Size,
			})
		}

	case pwr.ManifestMagic:
		{
			mh := &pwr.ManifestHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(mh))

			rctx, err = pwr.DecompressWire(rctx, mh.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))

			comm.Logf("%s: %s wharf manifest file (%s) with %s", path, prettySize, mh.GetCompression().ToString(), container.Stats())
			comm.Result(ContainerResult{
				Type:             "wharf/manifest",
				NumFiles:         len(container.Files),
				NumDirs:          len(container.Dirs),
				NumSymlinks:      len(container.Symlinks),
				UncompressedSize: container.Size,
			})
		}

	case pwr.WoundsMagic:
		{
			wh := &pwr.WoundsHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(wh))

			container := &tlc.Container{}
			must(rctx.ReadMessage(container))

			files := make(map[int64]bool)
			totalWounds := int64(0)

			for {
				wound := &pwr.Wound{}

				err = rctx.ReadMessage(wound)
				if err != nil {
					if errors.Is(err, io.EOF) {
						break
					} else {
						must(err)
					}
				}

				if wound.Kind == pwr.WoundKind_FILE {
					totalWounds += (wound.End - wound.Start)
					files[wound.Index] = true
				}
			}

			comm.Logf("%s: %s wharf wounds file with %s, %s wounds in %d files", path, prettySize, container.Stats(),
				humanize.IBytes(uint64(totalWounds)), len(files))
			comm.Result(ContainerResult{
				Type: "wharf/wounds",
			})
		}

	default:
		_, err := reader.Seek(0, os.SEEK_SET)
		must(err)

		wasZip := func() bool {
			zr, err := zip.NewReader(reader, stats.Size())
			if err != nil {
				if err != zip.ErrFormat {
					must(err)
				}
				return false
			}

			container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
			must(err)

			comm.Logf("%s: %s zip file with %s", path, prettySize, container.Stats())
			comm.Result(ContainerResult{
				Type:             "zip",
				NumFiles:         len(container.Files),
				NumDirs:          len(container.Dirs),
				NumSymlinks:      len(container.Symlinks),
				UncompressedSize: container.Size,
			})
			return true
		}()

		if !wasZip {
			comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
		}
	}
}
Exemplo n.º 6
0
func ls(path string) {
	reader, err := eos.Open(path)
	must(err)

	defer reader.Close()

	stats, err := reader.Stat()
	if os.IsNotExist(err) {
		comm.Dief("%s: no such file or directory", path)
	}
	must(err)

	if stats.IsDir() {
		comm.Logf("%s: directory", path)
		return
	}

	if stats.Size() == 0 {
		comm.Logf("%s: empty file. peaceful.", path)
		return
	}

	log := func(line string) {
		comm.Logf(line)
	}

	var magic int32
	must(binary.Read(reader, wire.Endianness, &magic))

	switch magic {
	case pwr.PatchMagic:
		{
			h := &pwr.PatchHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			log("pre-patch container:")
			container.Print(log)

			container.Reset()
			must(rctx.ReadMessage(container))
			log("================================")
			log("post-patch container:")
			container.Print(log)
		}

	case pwr.SignatureMagic:
		{
			h := &pwr.SignatureHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			container.Print(log)
		}

	case pwr.ManifestMagic:
		{
			h := &pwr.ManifestHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			container.Print(log)
		}

	case pwr.WoundsMagic:
		{
			wh := &pwr.WoundsHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(wh))

			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			container.Print(log)

			for {
				wound := &pwr.Wound{}
				err = rctx.ReadMessage(wound)
				if err != nil {
					if errors.Is(err, io.EOF) {
						break
					} else {
						must(err)
					}
				}
				comm.Logf(wound.PrettyString(container))
			}
		}

	default:
		_, err := reader.Seek(0, os.SEEK_SET)
		must(err)

		wasZip := func() bool {
			zr, err := zip.NewReader(reader, stats.Size())
			if err != nil {
				if err != zip.ErrFormat {
					must(err)
				}
				return false
			}

			container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
			must(err)
			container.Print(log)
			return true
		}()

		if !wasZip {
			comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
		}
	}
}