Example #1
0
func doSign(output string, signature string, compression pwr.CompressionSettings, fixPerms bool) error {
	comm.Opf("Creating signature for %s", output)
	startTime := time.Now()

	container, err := tlc.WalkAny(output, filterPaths)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	pool, err := pools.New(container, output)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	if fixPerms {
		container.FixPermissions(pool)
	}

	signatureWriter, err := os.Create(signature)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	rawSigWire := wire.NewWriteContext(signatureWriter)
	rawSigWire.WriteMagic(pwr.SignatureMagic)

	rawSigWire.WriteMessage(&pwr.SignatureHeader{
		Compression: &compression,
	})

	sigWire, err := pwr.CompressWire(rawSigWire, &compression)
	if err != nil {
		return errors.Wrap(err, 1)
	}
	sigWire.WriteMessage(container)

	comm.StartProgress()
	err = pwr.ComputeSignatureToWriter(container, pool, comm.NewStateConsumer(), func(hash wsync.BlockHash) error {
		return sigWire.WriteMessage(&pwr.BlockHash{
			WeakHash:   hash.WeakHash,
			StrongHash: hash.StrongHash,
		})
	})
	comm.EndProgress()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = sigWire.Close()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	prettySize := humanize.IBytes(uint64(container.Size))
	perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond)

	return nil
}
Example #2
0
// CompressWire wraps a wire.WriteContext into a compressor, according to given settings,
// so that any messages written through the returned WriteContext will first be compressed.
func CompressWire(ctx *wire.WriteContext, compression *CompressionSettings) (*wire.WriteContext, error) {
	if compression == nil {
		return nil, errors.Wrap(fmt.Errorf("no compression specified"), 1)
	}

	if compression.Algorithm == CompressionAlgorithm_NONE {
		return ctx, nil
	}

	compressor := compressors[compression.Algorithm]
	if compressor == nil {
		return nil, errors.Wrap(fmt.Errorf("no compressor registered for %s", compression.Algorithm.String()), 1)
	}

	compressedWriter, err := compressor.Apply(ctx.Writer(), compression.Quality)
	if err != nil {
		return nil, errors.Wrap(err, 1)
	}

	return wire.NewWriteContext(compressedWriter), nil
}
Example #3
0
func Test_Compression(t *testing.T) {
	fc := &fakeCompressor{}
	RegisterCompressor(CompressionAlgorithm_GZIP, fc)

	fd := &fakeDecompressor{}
	RegisterDecompressor(CompressionAlgorithm_GZIP, fd)

	assert.EqualValues(t, false, fc.called)

	buf := new(bytes.Buffer)
	wc := wire.NewWriteContext(buf)
	_, err := CompressWire(wc, &CompressionSettings{
		Algorithm: CompressionAlgorithm_BROTLI,
		Quality:   3,
	})
	assert.NotNil(t, err)

	cwc, err := CompressWire(wc, &CompressionSettings{
		Algorithm: CompressionAlgorithm_GZIP,
		Quality:   3,
	})
	assert.NoError(t, err)

	assert.EqualValues(t, true, fc.called)
	assert.EqualValues(t, 3, fc.quality)

	assert.NoError(t, cwc.WriteMessage(&SyncHeader{
		FileIndex: 672,
	}))

	rc := wire.NewReadContext(bytes.NewReader(buf.Bytes()))

	sh := &SyncHeader{}
	assert.NoError(t, rc.ReadMessage(sh))

	assert.EqualValues(t, 672, sh.FileIndex)
	assert.NotNil(t, rc.ReadMessage(sh))
}
Example #4
0
// Do only create a file at WoundsPath when it receives the first wound.
// If no wounds are ever received, Do will effectively be a no-op.
func (ww *WoundsWriter) Do(container *tlc.Container, wounds chan *Wound) error {
	var fw *os.File
	var wc *wire.WriteContext

	defer func() {
		if wc != nil {
			wc.Close()
		}

		if fw != nil {
			fw.Close()
		}
	}()

	writeWound := func(wound *Wound) error {
		ww.totalCorrupted += wound.Size()

		if wc == nil {
			var err error
			fw, err = os.Create(ww.WoundsPath)
			if err != nil {
				return errors.Wrap(err, 1)
			}

			wc = wire.NewWriteContext(fw)
			if err != nil {
				return errors.Wrap(err, 1)
			}

			err = wc.WriteMagic(WoundsMagic)
			if err != nil {
				return errors.Wrap(err, 1)
			}

			err = wc.WriteMessage(&WoundsHeader{})
			if err != nil {
				return errors.Wrap(err, 1)
			}

			err = wc.WriteMessage(container)
			if err != nil {
				return errors.Wrap(err, 1)
			}
		}

		err := wc.WriteMessage(wound)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		return nil
	}

	for wound := range wounds {
		if wound.Healthy() {
			continue
		}

		ww.hasWounds = true
		err := writeWound(wound)
		if err != nil {
			return err
		}
	}

	return nil
}
Example #5
0
func doCmdBsdiff(target string, source string, patch string, concurrency int, measureOverhead bool) error {
	targetReader, err := os.Open(target)
	if err != nil {
		return err
	}

	defer targetReader.Close()

	targetStats, err := targetReader.Stat()
	if err != nil {
		return err
	}

	sourceReader, err := os.Open(source)
	if err != nil {
		return err
	}

	defer sourceReader.Close()

	sourceStats, err := sourceReader.Stat()
	if err != nil {
		return err
	}

	comm.Opf("Diffing %s (%s) and %s (%s)...", target, humanize.IBytes(uint64(targetStats.Size())), source,
		humanize.IBytes(uint64(sourceStats.Size())))

	patchWriter, err := os.Create(patch)
	if err != nil {
		return err
	}

	wctx := wire.NewWriteContext(patchWriter)

	err = wctx.WriteMagic(pwr.PatchMagic)
	if err != nil {
		return err
	}

	compression := butlerCompressionSettings()

	err = wctx.WriteMessage(&pwr.PatchHeader{
		Compression: &compression,
	})
	if err != nil {
		return err
	}

	wctx, err = pwr.CompressWire(wctx, &compression)
	if err != nil {
		return err
	}

	targetContainer := &tlc.Container{}
	targetContainer.Files = []*tlc.File{
		&tlc.File{
			Path: target,
			Size: targetStats.Size(),
		},
	}

	err = wctx.WriteMessage(targetContainer)
	if err != nil {
		return err
	}

	sourceContainer := &tlc.Container{}
	sourceContainer.Files = []*tlc.File{
		&tlc.File{
			Path: source,
			Size: sourceStats.Size(),
		},
	}

	err = wctx.WriteMessage(sourceContainer)
	if err != nil {
		return err
	}

	err = wctx.WriteMessage(&pwr.SyncHeader{
		FileIndex: 0,
	})
	if err != nil {
		return err
	}

	err = wctx.WriteMessage(&pwr.SyncOp{
		Type:      pwr.SyncOp_BSDIFF,
		FileIndex: 0,
	})
	if err != nil {
		return err
	}

	startTime := time.Now()

	comm.StartProgress()

	dc := bsdiff.DiffContext{
		MeasureMem:              *appArgs.memstats,
		MeasureParallelOverhead: measureOverhead,
		SuffixSortConcurrency:   concurrency,
	}

	err = dc.Do(targetReader, sourceReader, wctx.WriteMessage, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	comm.EndProgress()

	err = wctx.WriteMessage(&pwr.SyncOp{
		Type: pwr.SyncOp_HEY_YOU_DID_IT,
	})
	if err != nil {
		return err
	}

	err = wctx.Close()
	if err != nil {
		return err
	}

	patchStats, err := os.Lstat(patch)
	if err != nil {
		return err
	}

	duration := time.Since(startTime)
	perSec := float64(sourceStats.Size()) / duration.Seconds()

	relToNew := 100.0 * float64(patchStats.Size()) / float64(sourceStats.Size())

	comm.Statf("Processed %s @ %s / s, total %s", humanize.IBytes(uint64(sourceStats.Size())), humanize.IBytes(uint64(perSec)), duration)
	comm.Statf("Wrote %s patch (%.2f%% of total size) to %s", humanize.IBytes(uint64(patchStats.Size())), relToNew, patch)

	return nil
}
Example #6
0
// WriteManifest writes container info and block addresses in wharf's manifest format
// Does not close manifestWriter.
func WriteManifest(manifestWriter io.Writer, compression *pwr.CompressionSettings, container *tlc.Container, blockHashes *BlockHashMap) error {
	rawWire := wire.NewWriteContext(manifestWriter)
	err := rawWire.WriteMagic(pwr.ManifestMagic)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = rawWire.WriteMessage(&pwr.ManifestHeader{
		Compression: compression,
		Algorithm:   pwr.HashAlgorithm_SHAKE128_32,
	})
	if err != nil {
		return errors.Wrap(err, 1)
	}

	wire, err := pwr.CompressWire(rawWire, compression)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = wire.WriteMessage(container)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	sh := &pwr.SyncHeader{}
	mbh := &pwr.ManifestBlockHash{}

	for fileIndex, f := range container.Files {
		sh.Reset()
		sh.FileIndex = int64(fileIndex)
		err = wire.WriteMessage(sh)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		numBlocks := ComputeNumBlocks(f.Size)

		for blockIndex := int64(0); blockIndex < numBlocks; blockIndex++ {
			loc := BlockLocation{FileIndex: int64(fileIndex), BlockIndex: blockIndex}
			hash := blockHashes.Get(loc)
			if hash == nil {
				err = fmt.Errorf("missing BlockHash for block %+v", loc)
				return errors.Wrap(err, 1)
			}

			mbh.Reset()
			mbh.Hash = hash

			err = wire.WriteMessage(mbh)
			if err != nil {
				return errors.Wrap(err, 1)
			}
		}
	}

	err = wire.Close()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	return nil
}
Example #7
0
File: diff.go Project: itchio/wharf
// WritePatch outputs a pwr patch to patchWriter
func (dctx *DiffContext) WritePatch(patchWriter io.Writer, signatureWriter io.Writer) error {
	if dctx.Compression == nil {
		return errors.Wrap(fmt.Errorf("No compression settings specified, bailing out"), 1)
	}

	// signature header
	rawSigWire := wire.NewWriteContext(signatureWriter)
	err := rawSigWire.WriteMagic(SignatureMagic)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = rawSigWire.WriteMessage(&SignatureHeader{
		Compression: dctx.Compression,
	})
	if err != nil {
		return errors.Wrap(err, 1)
	}

	sigWire, err := CompressWire(rawSigWire, dctx.Compression)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = sigWire.WriteMessage(dctx.SourceContainer)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	// patch header
	rawPatchWire := wire.NewWriteContext(patchWriter)
	err = rawPatchWire.WriteMagic(PatchMagic)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	header := &PatchHeader{
		Compression: dctx.Compression,
	}

	err = rawPatchWire.WriteMessage(header)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	patchWire, err := CompressWire(rawPatchWire, dctx.Compression)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = patchWire.WriteMessage(dctx.TargetContainer)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = patchWire.WriteMessage(dctx.SourceContainer)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	sourceBytes := dctx.SourceContainer.Size
	fileOffset := int64(0)

	onSourceRead := func(count int64) {
		dctx.Consumer.Progress(float64(fileOffset+count) / float64(sourceBytes))
	}

	sigWriter := makeSigWriter(sigWire)
	opsWriter := makeOpsWriter(patchWire, dctx)

	diffContext := mksync()
	signContext := mksync()
	blockLibrary := wsync.NewBlockLibrary(dctx.TargetSignature)

	targetContainerPathToIndex := make(map[string]int64)
	for index, f := range dctx.TargetContainer.Files {
		targetContainerPathToIndex[f.Path] = int64(index)
	}

	// re-used messages
	syncHeader := &SyncHeader{}
	syncDelimiter := &SyncOp{
		Type: SyncOp_HEY_YOU_DID_IT,
	}

	pool := dctx.Pool
	defer func() {
		if fErr := pool.Close(); fErr != nil && err == nil {
			err = errors.Wrap(fErr, 1)
		}
	}()

	for fileIndex, f := range dctx.SourceContainer.Files {
		dctx.Consumer.ProgressLabel(f.Path)
		dctx.Consumer.Debug(fmt.Sprintf("%s (%s)", f.Path, humanize.IBytes(uint64(f.Size))))
		fileOffset = f.Offset

		syncHeader.Reset()
		syncHeader.FileIndex = int64(fileIndex)
		err = patchWire.WriteMessage(syncHeader)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		var sourceReader io.Reader
		sourceReader, err = pool.GetReader(int64(fileIndex))
		if err != nil {
			return errors.Wrap(err, 1)
		}

		//             / differ
		// source file +
		//             \ signer
		diffReader, diffWriter := io.Pipe()
		signReader, signWriter := io.Pipe()

		done := make(chan bool)
		errs := make(chan error)

		var preferredFileIndex int64 = -1
		if oldIndex, ok := targetContainerPathToIndex[f.Path]; ok {
			preferredFileIndex = oldIndex
		}

		go diffFile(diffContext, dctx, blockLibrary, diffReader, opsWriter, preferredFileIndex, errs, done)
		go signFile(signContext, fileIndex, signReader, sigWriter, errs, done)

		go func() {
			defer func() {
				if dErr := diffWriter.Close(); dErr != nil {
					errs <- errors.Wrap(dErr, 1)
				}
			}()
			defer func() {
				if sErr := signWriter.Close(); sErr != nil {
					errs <- errors.Wrap(sErr, 1)
				}
			}()

			mw := io.MultiWriter(diffWriter, signWriter)

			sourceReadCounter := counter.NewReaderCallback(onSourceRead, sourceReader)
			_, cErr := io.Copy(mw, sourceReadCounter)
			if cErr != nil {
				errs <- errors.Wrap(cErr, 1)
			}
		}()

		// wait until all are done
		// or an error occurs
		for c := 0; c < 2; c++ {
			select {
			case wErr := <-errs:
				return errors.Wrap(wErr, 1)
			case <-done:
			}
		}

		err = patchWire.WriteMessage(syncDelimiter)
		if err != nil {
			return errors.Wrap(err, 1)
		}
	}

	err = patchWire.Close()
	if err != nil {
		return errors.Wrap(err, 1)
	}
	err = sigWire.Close()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	return nil
}