func Test_Writer(t *testing.T) {
dropSize := 16
numShort := 0
shortSize := 0
validate := func(buf []byte) error {
switch {
case len(buf) == dropSize:
if numShort > 0 {
return fmt.Errorf("got full after short")
}
return nil
case len(buf) < dropSize:
if numShort > 0 {
return fmt.Errorf("got second short (%d)", len(buf))
}
numShort++
shortSize = len(buf)
default:
return fmt.Errorf("drop too large (%d > %d)", len(buf), dropSize)
}
return nil
}
buf := make([]byte, dropSize)
countingWriter := counter.NewWriter(nil)
dw := &Writer{
Buffer: buf,
Validate: validate,
Writer: countingWriter,
}
rbuf := make([]byte, 128)
write := func(l int) {
written, wErr := dw.Write(rbuf[0:l])
assert.Equal(t, l, written)
assert.NoError(t, wErr)
}
write(12)
write(4)
write(10)
write(6)
write(16)
write(64)
write(5)
assert.NoError(t, dw.Close())
assert.Equal(t, 5, shortSize)
assert.Equal(t, int64(12+4+10+6+16+64+5), countingWriter.Count())
}
func doPush(buildPath string, specStr string, userVersion string, fixPerms bool) error {
// start walking source container while waiting on auth flow
sourceContainerChan := make(chan walkResult)
walkErrs := make(chan error)
go doWalk(buildPath, sourceContainerChan, walkErrs, fixPerms)
spec, err := itchio.ParseSpec(specStr)
if err != nil {
return errors.Wrap(err, 1)
}
err = spec.EnsureChannel()
if err != nil {
return errors.Wrap(err, 1)
}
client, err := authenticateViaOauth()
if err != nil {
return errors.Wrap(err, 1)
}
newBuildRes, err := client.CreateBuild(spec.Target, spec.Channel, userVersion)
if err != nil {
return errors.Wrap(err, 1)
}
buildID := newBuildRes.Build.ID
parentID := newBuildRes.Build.ParentBuild.ID
var targetSignature *pwr.SignatureInfo
if parentID == 0 {
comm.Opf("For channel `%s`: pushing first build", spec.Channel)
targetSignature = &pwr.SignatureInfo{
Container: &tlc.Container{},
Hashes: make([]wsync.BlockHash, 0),
}
} else {
comm.Opf("For channel `%s`: last build is %d, downloading its signature", spec.Channel, parentID)
var buildFiles itchio.ListBuildFilesResponse
buildFiles, err = client.ListBuildFiles(parentID)
if err != nil {
return errors.Wrap(err, 1)
}
signatureFile := itchio.FindBuildFile(itchio.BuildFileType_SIGNATURE, buildFiles.Files)
if signatureFile == nil {
comm.Dief("Could not find signature for parent build %d, aborting", parentID)
}
var signatureReader io.Reader
signatureReader, err = client.DownloadBuildFile(parentID, signatureFile.ID)
if err != nil {
return errors.Wrap(err, 1)
}
targetSignature, err = pwr.ReadSignature(signatureReader)
if err != nil {
return errors.Wrap(err, 1)
}
}
newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)
if err != nil {
return errors.Wrap(err, 1)
}
uploadDone := make(chan bool)
uploadErrs := make(chan error)
patchWriter, err := uploader.NewResumableUpload(newPatchRes.File.UploadURL,
uploadDone, uploadErrs, uploader.ResumableUploadSettings{
Consumer: comm.NewStateConsumer(),
})
patchWriter.MaxChunkGroup = *appArgs.maxChunkGroup
if err != nil {
return errors.Wrap(err, 1)
}
signatureWriter, err := uploader.NewResumableUpload(newSignatureRes.File.UploadURL,
uploadDone, uploadErrs, uploader.ResumableUploadSettings{
Consumer: comm.NewStateConsumer(),
})
signatureWriter.MaxChunkGroup = *appArgs.maxChunkGroup
if err != nil {
return errors.Wrap(err, 1)
}
comm.Debugf("Launching patch & signature channels")
patchCounter := counter.NewWriter(patchWriter)
signatureCounter := counter.NewWriter(signatureWriter)
// we started walking the source container in the beginning,
// we actually need it now.
// note that we could actually start diffing before all the file
// creation & upload setup is done
var sourceContainer *tlc.Container
var sourcePool wsync.Pool
comm.Debugf("Waiting for source container")
select {
case walkErr := <-walkErrs:
return errors.Wrap(walkErr, 1)
case walkies := <-sourceContainerChan:
comm.Debugf("Got sourceContainer!")
sourceContainer = walkies.container
sourcePool = walkies.pool
break
}
comm.Opf("Pushing %s (%s)", humanize.IBytes(uint64(sourceContainer.Size)), sourceContainer.Stats())
comm.Debugf("Building diff context")
var readBytes int64
bytesPerSec := float64(0)
lastUploadedBytes := int64(0)
stopTicking := make(chan struct{})
updateProgress := func() {
uploadedBytes := int64(float64(patchWriter.UploadedBytes))
// input bytes that aren't in output, for example:
// - bytes that have been compressed away
// - bytes that were in old build and were simply reused
goneBytes := readBytes - patchWriter.TotalBytes
conservativeTotalBytes := sourceContainer.Size - goneBytes
leftBytes := conservativeTotalBytes - uploadedBytes
if leftBytes > AlmostThereThreshold {
netStatus := "- network idle"
if bytesPerSec > 1 {
netStatus = fmt.Sprintf("@ %s/s", humanize.IBytes(uint64(bytesPerSec)))
}
comm.ProgressLabel(fmt.Sprintf("%s, %s left", netStatus, humanize.IBytes(uint64(leftBytes))))
} else {
comm.ProgressLabel(fmt.Sprintf("- almost there"))
}
conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
conservativeProgress = min(1.0, conservativeProgress)
comm.Progress(conservativeProgress)
comm.ProgressScale(float64(readBytes) / float64(sourceContainer.Size))
}
go func() {
ticker := time.NewTicker(time.Second * time.Duration(2))
for {
select {
case <-ticker.C:
bytesPerSec = float64(patchWriter.UploadedBytes-lastUploadedBytes) / 2.0
lastUploadedBytes = patchWriter.UploadedBytes
updateProgress()
case <-stopTicking:
break
}
}
}()
patchWriter.OnProgress = updateProgress
stateConsumer := &state.Consumer{
OnProgress: func(progress float64) {
readBytes = int64(float64(sourceContainer.Size) * progress)
updateProgress()
},
}
dctx := &pwr.DiffContext{
Compression: &pwr.CompressionSettings{
Algorithm: pwr.CompressionAlgorithm_BROTLI,
Quality: 1,
},
SourceContainer: sourceContainer,
Pool: sourcePool,
TargetContainer: targetSignature.Container,
TargetSignature: targetSignature.Hashes,
Consumer: stateConsumer,
}
comm.StartProgress()
comm.ProgressScale(0.0)
err = dctx.WritePatch(patchCounter, signatureCounter)
if err != nil {
return errors.Wrap(err, 1)
}
// close in a goroutine to avoid deadlocking
doClose := func(c io.Closer, done chan bool, errs chan error) {
closeErr := c.Close()
if closeErr != nil {
errs <- errors.Wrap(closeErr, 1)
return
}
done <- true
}
go doClose(patchWriter, uploadDone, uploadErrs)
go doClose(signatureWriter, uploadDone, uploadErrs)
for c := 0; c < 4; c++ {
select {
case uploadErr := <-uploadErrs:
return errors.Wrap(uploadErr, 1)
case <-uploadDone:
comm.Debugf("upload done")
}
}
close(stopTicking)
comm.ProgressLabel("finalizing build")
finalDone := make(chan bool)
finalErrs := make(chan error)
doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
if err != nil {
errs <- errors.Wrap(err, 1)
return
}
done <- true
}
go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)
for i := 0; i < 2; i++ {
select {
case err := <-finalErrs:
return errors.Wrap(err, 1)
case <-finalDone:
}
}
comm.EndProgress()
{
prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count()))
percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes))
savings := 100.0 - relToNew
if dctx.ReusedBytes > 0 {
comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
} else {
comm.Statf("Added %s fresh data", prettyFreshSize)
}
if savings > 0 && !math.IsNaN(savings) {
comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
} else {
comm.Statf("%s patch (no savings)", prettyPatchSize)
}
}
comm.Opf("Build is now processing, should be up in a bit (see `butler status`)")
comm.Logf("")
return nil
}
func CompressZip(archiveWriter io.Writer, dir string, consumer *state.Consumer) (*CompressResult, error) {
var err error
var uncompressedSize int64
var compressedSize int64
archiveCounter := counter.NewWriter(archiveWriter)
zipWriter := zip.NewWriter(archiveCounter)
defer zipWriter.Close()
defer func() {
if zipWriter != nil {
if zErr := zipWriter.Close(); err == nil && zErr != nil {
err = errors.Wrap(zErr, 1)
}
}
}()
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
name, wErr := filepath.Rel(dir, path)
if wErr != nil {
return wErr
}
if name == "." {
// don't add '.' to zip
return nil
}
name = filepath.ToSlash(name)
fh, wErr := zip.FileInfoHeader(info)
if wErr != nil {
return wErr
}
fh.Name = name
writer, wErr := zipWriter.CreateHeader(fh)
if wErr != nil {
return wErr
}
if info.IsDir() {
// good!
} else if info.Mode()&os.ModeSymlink > 0 {
dest, wErr := os.Readlink(path)
if wErr != nil {
return wErr
}
_, wErr = writer.Write([]byte(dest))
if wErr != nil {
return wErr
}
} else if info.Mode().IsRegular() {
reader, wErr := os.Open(path)
if wErr != nil {
return wErr
}
defer reader.Close()
copiedBytes, wErr := io.Copy(writer, reader)
if wErr != nil {
return wErr
}
uncompressedSize += copiedBytes
}
return nil
})
err = zipWriter.Close()
if err != nil {
return nil, errors.Wrap(err, 1)
}
zipWriter = nil
compressedSize = archiveCounter.Count()
return &CompressResult{
UncompressedSize: uncompressedSize,
CompressedSize: compressedSize,
}, err
}
func CompressZip(archiveWriter io.Writer, container *tlc.Container, pool wsync.Pool, consumer *state.Consumer) (*archiver.CompressResult, error) {
var err error
var uncompressedSize int64
var compressedSize int64
archiveCounter := counter.NewWriter(archiveWriter)
zipWriter := zip.NewWriter(archiveCounter)
defer zipWriter.Close()
defer func() {
if zipWriter != nil {
if zErr := zipWriter.Close(); err == nil && zErr != nil {
err = errors.Wrap(zErr, 1)
}
}
}()
for _, dir := range container.Dirs {
fh := zip.FileHeader{
Name: dir.Path + "/",
}
fh.SetMode(os.FileMode(dir.Mode))
fh.SetModTime(time.Now())
_, hErr := zipWriter.CreateHeader(&fh)
if hErr != nil {
return nil, errors.Wrap(hErr, 1)
}
}
for fileIndex, file := range container.Files {
fh := zip.FileHeader{
Name: file.Path,
UncompressedSize64: uint64(file.Size),
Method: zip.Deflate,
}
fh.SetMode(os.FileMode(file.Mode))
fh.SetModTime(time.Now())
entryWriter, eErr := zipWriter.CreateHeader(&fh)
if eErr != nil {
return nil, errors.Wrap(eErr, 1)
}
entryReader, eErr := pool.GetReader(int64(fileIndex))
if eErr != nil {
return nil, errors.Wrap(eErr, 1)
}
copiedBytes, eErr := io.Copy(entryWriter, entryReader)
if eErr != nil {
return nil, errors.Wrap(eErr, 1)
}
uncompressedSize += copiedBytes
}
for _, symlink := range container.Symlinks {
fh := zip.FileHeader{
Name: symlink.Path,
}
fh.SetMode(os.FileMode(symlink.Mode))
entryWriter, eErr := zipWriter.CreateHeader(&fh)
if eErr != nil {
return nil, errors.Wrap(eErr, 1)
}
entryWriter.Write([]byte(symlink.Dest))
}
err = zipWriter.Close()
if err != nil {
return nil, errors.Wrap(err, 1)
}
zipWriter = nil
compressedSize = archiveCounter.Count()
return &archiver.CompressResult{
UncompressedSize: uncompressedSize,
CompressedSize: compressedSize,
}, nil
}
func doDiff(target string, source string, patch string, compression pwr.CompressionSettings) error {
var err error
startTime := time.Now()
targetSignature := &pwr.SignatureInfo{}
targetSignature.Container, err = tlc.WalkAny(target, filterPaths)
if err != nil {
// Signature file perhaps?
var signatureReader io.ReadCloser
signatureReader, err = eos.Open(target)
if err != nil {
return errors.Wrap(err, 1)
}
targetSignature, err = pwr.ReadSignature(signatureReader)
if err != nil {
if errors.Is(err, wire.ErrFormat) {
return fmt.Errorf("unrecognized target %s (not a container, not a signature file)", target)
}
return errors.Wrap(err, 1)
}
comm.Opf("Read signature from %s", target)
err = signatureReader.Close()
if err != nil {
return errors.Wrap(err, 1)
}
} else {
// Container (dir, archive, etc.)
comm.Opf("Hashing %s", target)
comm.StartProgress()
var targetPool wsync.Pool
targetPool, err = pools.New(targetSignature.Container, target)
if err != nil {
return errors.Wrap(err, 1)
}
targetSignature.Hashes, err = pwr.ComputeSignature(targetSignature.Container, targetPool, comm.NewStateConsumer())
comm.EndProgress()
if err != nil {
return errors.Wrap(err, 1)
}
{
prettySize := humanize.IBytes(uint64(targetSignature.Container.Size))
perSecond := humanize.IBytes(uint64(float64(targetSignature.Container.Size) / time.Since(startTime).Seconds()))
comm.Statf("%s (%s) @ %s/s\n", prettySize, targetSignature.Container.Stats(), perSecond)
}
}
startTime = time.Now()
var sourceContainer *tlc.Container
sourceContainer, err = tlc.WalkAny(source, filterPaths)
if err != nil {
return errors.Wrap(err, 1)
}
var sourcePool wsync.Pool
sourcePool, err = pools.New(sourceContainer, source)
if err != nil {
return errors.Wrap(err, 1)
}
patchWriter, err := os.Create(patch)
if err != nil {
return errors.Wrap(err, 1)
}
defer patchWriter.Close()
signaturePath := patch + ".sig"
signatureWriter, err := os.Create(signaturePath)
if err != nil {
return errors.Wrap(err, 1)
}
defer signatureWriter.Close()
patchCounter := counter.NewWriter(patchWriter)
signatureCounter := counter.NewWriter(signatureWriter)
dctx := &pwr.DiffContext{
SourceContainer: sourceContainer,
Pool: sourcePool,
TargetContainer: targetSignature.Container,
TargetSignature: targetSignature.Hashes,
Consumer: comm.NewStateConsumer(),
Compression: &compression,
}
comm.Opf("Diffing %s", source)
comm.StartProgress()
err = dctx.WritePatch(patchCounter, signatureCounter)
if err != nil {
return errors.Wrap(err, 1)
}
comm.EndProgress()
totalDuration := time.Since(startTime)
{
prettySize := humanize.IBytes(uint64(sourceContainer.Size))
perSecond := humanize.IBytes(uint64(float64(sourceContainer.Size) / totalDuration.Seconds()))
comm.Statf("%s (%s) @ %s/s\n", prettySize, sourceContainer.Stats(), perSecond)
}
if *diffArgs.verify {
tmpDir, err := ioutil.TempDir("", "pwr")
if err != nil {
return errors.Wrap(err, 1)
}
defer os.RemoveAll(tmpDir)
apply(patch, target, tmpDir, false, signaturePath, "")
}
{
prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count()))
percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes))
comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
comm.Statf("%s patch (%.2f%% of the full size) in %s", prettyPatchSize, relToNew, totalDuration)
}
return nil
}