Example #1
0
func doSign(output string, signature string, compression pwr.CompressionSettings, fixPerms bool) error {
	comm.Opf("Creating signature for %s", output)
	startTime := time.Now()

	container, err := tlc.WalkAny(output, filterPaths)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	pool, err := pools.New(container, output)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	if fixPerms {
		container.FixPermissions(pool)
	}

	signatureWriter, err := os.Create(signature)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	rawSigWire := wire.NewWriteContext(signatureWriter)
	rawSigWire.WriteMagic(pwr.SignatureMagic)

	rawSigWire.WriteMessage(&pwr.SignatureHeader{
		Compression: &compression,
	})

	sigWire, err := pwr.CompressWire(rawSigWire, &compression)
	if err != nil {
		return errors.Wrap(err, 1)
	}
	sigWire.WriteMessage(container)

	comm.StartProgress()
	err = pwr.ComputeSignatureToWriter(container, pool, comm.NewStateConsumer(), func(hash wsync.BlockHash) error {
		return sigWire.WriteMessage(&pwr.BlockHash{
			WeakHash:   hash.WeakHash,
			StrongHash: hash.StrongHash,
		})
	})
	comm.EndProgress()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = sigWire.Close()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	prettySize := humanize.IBytes(uint64(container.Size))
	perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond)

	return nil
}
Example #2
0
func doCmdBsdiff(target string, source string, patch string, concurrency int, measureOverhead bool) error {
	targetReader, err := os.Open(target)
	if err != nil {
		return err
	}

	defer targetReader.Close()

	targetStats, err := targetReader.Stat()
	if err != nil {
		return err
	}

	sourceReader, err := os.Open(source)
	if err != nil {
		return err
	}

	defer sourceReader.Close()

	sourceStats, err := sourceReader.Stat()
	if err != nil {
		return err
	}

	comm.Opf("Diffing %s (%s) and %s (%s)...", target, humanize.IBytes(uint64(targetStats.Size())), source,
		humanize.IBytes(uint64(sourceStats.Size())))

	patchWriter, err := os.Create(patch)
	if err != nil {
		return err
	}

	wctx := wire.NewWriteContext(patchWriter)

	err = wctx.WriteMagic(pwr.PatchMagic)
	if err != nil {
		return err
	}

	compression := butlerCompressionSettings()

	err = wctx.WriteMessage(&pwr.PatchHeader{
		Compression: &compression,
	})
	if err != nil {
		return err
	}

	wctx, err = pwr.CompressWire(wctx, &compression)
	if err != nil {
		return err
	}

	targetContainer := &tlc.Container{}
	targetContainer.Files = []*tlc.File{
		&tlc.File{
			Path: target,
			Size: targetStats.Size(),
		},
	}

	err = wctx.WriteMessage(targetContainer)
	if err != nil {
		return err
	}

	sourceContainer := &tlc.Container{}
	sourceContainer.Files = []*tlc.File{
		&tlc.File{
			Path: source,
			Size: sourceStats.Size(),
		},
	}

	err = wctx.WriteMessage(sourceContainer)
	if err != nil {
		return err
	}

	err = wctx.WriteMessage(&pwr.SyncHeader{
		FileIndex: 0,
	})
	if err != nil {
		return err
	}

	err = wctx.WriteMessage(&pwr.SyncOp{
		Type:      pwr.SyncOp_BSDIFF,
		FileIndex: 0,
	})
	if err != nil {
		return err
	}

	startTime := time.Now()

	comm.StartProgress()

	dc := bsdiff.DiffContext{
		MeasureMem:              *appArgs.memstats,
		MeasureParallelOverhead: measureOverhead,
		SuffixSortConcurrency:   concurrency,
	}

	err = dc.Do(targetReader, sourceReader, wctx.WriteMessage, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	comm.EndProgress()

	err = wctx.WriteMessage(&pwr.SyncOp{
		Type: pwr.SyncOp_HEY_YOU_DID_IT,
	})
	if err != nil {
		return err
	}

	err = wctx.Close()
	if err != nil {
		return err
	}

	patchStats, err := os.Lstat(patch)
	if err != nil {
		return err
	}

	duration := time.Since(startTime)
	perSec := float64(sourceStats.Size()) / duration.Seconds()

	relToNew := 100.0 * float64(patchStats.Size()) / float64(sourceStats.Size())

	comm.Statf("Processed %s @ %s / s, total %s", humanize.IBytes(uint64(sourceStats.Size())), humanize.IBytes(uint64(perSec)), duration)
	comm.Statf("Wrote %s patch (%.2f%% of total size) to %s", humanize.IBytes(uint64(patchStats.Size())), relToNew, patch)

	return nil
}
Example #3
0
// WriteManifest writes container info and block addresses in wharf's manifest format
// Does not close manifestWriter.
func WriteManifest(manifestWriter io.Writer, compression *pwr.CompressionSettings, container *tlc.Container, blockHashes *BlockHashMap) error {
	rawWire := wire.NewWriteContext(manifestWriter)
	err := rawWire.WriteMagic(pwr.ManifestMagic)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = rawWire.WriteMessage(&pwr.ManifestHeader{
		Compression: compression,
		Algorithm:   pwr.HashAlgorithm_SHAKE128_32,
	})
	if err != nil {
		return errors.Wrap(err, 1)
	}

	wire, err := pwr.CompressWire(rawWire, compression)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = wire.WriteMessage(container)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	sh := &pwr.SyncHeader{}
	mbh := &pwr.ManifestBlockHash{}

	for fileIndex, f := range container.Files {
		sh.Reset()
		sh.FileIndex = int64(fileIndex)
		err = wire.WriteMessage(sh)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		numBlocks := ComputeNumBlocks(f.Size)

		for blockIndex := int64(0); blockIndex < numBlocks; blockIndex++ {
			loc := BlockLocation{FileIndex: int64(fileIndex), BlockIndex: blockIndex}
			hash := blockHashes.Get(loc)
			if hash == nil {
				err = fmt.Errorf("missing BlockHash for block %+v", loc)
				return errors.Wrap(err, 1)
			}

			mbh.Reset()
			mbh.Hash = hash

			err = wire.WriteMessage(mbh)
			if err != nil {
				return errors.Wrap(err, 1)
			}
		}
	}

	err = wire.Close()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	return nil
}