func doVerify(signaturePath string, dir string, woundsPath string, healPath string) error { if woundsPath == "" { if healPath == "" { comm.Opf("Verifying %s", dir) } else { comm.Opf("Verifying %s, healing as we go", dir) } } else { if healPath == "" { comm.Opf("Verifying %s, writing wounds to %s", dir, woundsPath) } else { comm.Dief("Options --wounds and --heal cannot be used at the same time") } } startTime := time.Now() signatureReader, err := eos.Open(signaturePath) if err != nil { return errors.Wrap(err, 1) } defer signatureReader.Close() signature, err := pwr.ReadSignature(signatureReader) if err != nil { return errors.Wrap(err, 1) } vc := &pwr.ValidatorContext{ Consumer: comm.NewStateConsumer(), WoundsPath: woundsPath, HealPath: healPath, } comm.StartProgressWithTotalBytes(signature.Container.Size) err = vc.Validate(dir, signature) if err != nil { return errors.Wrap(err, 1) } comm.EndProgress() prettySize := humanize.IBytes(uint64(signature.Container.Size)) perSecond := humanize.IBytes(uint64(float64(signature.Container.Size) / time.Since(startTime).Seconds())) comm.Statf("%s (%s) @ %s/s\n", prettySize, signature.Container.Stats(), perSecond) if vc.WoundsConsumer.HasWounds() { if healer, ok := vc.WoundsConsumer.(pwr.Healer); ok { comm.Statf("%s corrupted data found, %s healed", humanize.IBytes(uint64(vc.WoundsConsumer.TotalCorrupted())), humanize.IBytes(uint64(healer.TotalHealed()))) } else { comm.Dief("%s corrupted data found", humanize.IBytes(uint64(vc.WoundsConsumer.TotalCorrupted()))) } } return nil }
func doSign(output string, signature string, compression pwr.CompressionSettings, fixPerms bool) error { comm.Opf("Creating signature for %s", output) startTime := time.Now() container, err := tlc.WalkAny(output, filterPaths) if err != nil { return errors.Wrap(err, 1) } pool, err := pools.New(container, output) if err != nil { return errors.Wrap(err, 1) } if fixPerms { container.FixPermissions(pool) } signatureWriter, err := os.Create(signature) if err != nil { return errors.Wrap(err, 1) } rawSigWire := wire.NewWriteContext(signatureWriter) rawSigWire.WriteMagic(pwr.SignatureMagic) rawSigWire.WriteMessage(&pwr.SignatureHeader{ Compression: &compression, }) sigWire, err := pwr.CompressWire(rawSigWire, &compression) if err != nil { return errors.Wrap(err, 1) } sigWire.WriteMessage(container) comm.StartProgress() err = pwr.ComputeSignatureToWriter(container, pool, comm.NewStateConsumer(), func(hash wsync.BlockHash) error { return sigWire.WriteMessage(&pwr.BlockHash{ WeakHash: hash.WeakHash, StrongHash: hash.StrongHash, }) }) comm.EndProgress() if err != nil { return errors.Wrap(err, 1) } err = sigWire.Close() if err != nil { return errors.Wrap(err, 1) } prettySize := humanize.IBytes(uint64(container.Size)) perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds())) comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond) return nil }
func unzip(file string, dir string, resumeFile string) { comm.Opf("Extracting zip %s to %s", file, dir) settings := archiver.ExtractSettings{ Consumer: comm.NewStateConsumer(), ResumeFrom: resumeFile, OnUncompressedSizeKnown: func(uncompressedSize int64) { comm.StartProgressWithTotalBytes(uncompressedSize) }, } res, err := archiver.ExtractPath(file, dir, settings) comm.EndProgress() must(err) comm.Logf("Extracted %d dirs, %d files, %d symlinks", res.Dirs, res.Files, res.Symlinks) }
func doUpgrade(head bool) error { if head { if !comm.YesNo("Do you want to upgrade to the bleeding-edge version? Things may break!") { comm.Logf("Okay, not upgrading. Bye!") return nil } return applyUpgrade("head", "head") } if version == "head" { comm.Statf("Bleeding-edge, not upgrading unless told to.") comm.Logf("(Use `--head` if you want to upgrade to the latest bleeding-edge version)") return nil } comm.Opf("Looking for upgrades...") currentVer, latestVer, err := queryLatestVersion() if err != nil { return fmt.Errorf("Version check failed: %s", err.Error()) } if latestVer == nil || currentVer.GTE(*latestVer) { comm.Statf("Your butler is up-to-date. Have a nice day!") return nil } comm.Statf("Current version: %s", currentVer.String()) comm.Statf("Latest version : %s", latestVer.String()) if !comm.YesNo("Do you want to upgrade now?") { comm.Logf("Okay, not upgrading. Bye!") return nil } must(applyUpgrade(currentVer.String(), latestVer.String())) return nil }
func doPush(buildPath string, specStr string, userVersion string, fixPerms bool) error { // start walking source container while waiting on auth flow sourceContainerChan := make(chan walkResult) walkErrs := make(chan error) go doWalk(buildPath, sourceContainerChan, walkErrs, fixPerms) spec, err := itchio.ParseSpec(specStr) if err != nil { return errors.Wrap(err, 1) } err = spec.EnsureChannel() if err != nil { return errors.Wrap(err, 1) } client, err := authenticateViaOauth() if err != nil { return errors.Wrap(err, 1) } newBuildRes, err := client.CreateBuild(spec.Target, spec.Channel, userVersion) if err != nil { return errors.Wrap(err, 1) } buildID := newBuildRes.Build.ID parentID := newBuildRes.Build.ParentBuild.ID var targetSignature *pwr.SignatureInfo if parentID == 0 { comm.Opf("For channel `%s`: pushing first build", spec.Channel) targetSignature = &pwr.SignatureInfo{ Container: &tlc.Container{}, Hashes: make([]wsync.BlockHash, 0), } } else { comm.Opf("For channel `%s`: last build is %d, downloading its signature", spec.Channel, parentID) var buildFiles itchio.ListBuildFilesResponse buildFiles, err = client.ListBuildFiles(parentID) if err != nil { return errors.Wrap(err, 1) } signatureFile := itchio.FindBuildFile(itchio.BuildFileType_SIGNATURE, buildFiles.Files) if signatureFile == nil { comm.Dief("Could not find signature for parent build %d, aborting", parentID) } var signatureReader io.Reader signatureReader, err = client.DownloadBuildFile(parentID, signatureFile.ID) if err != nil { return errors.Wrap(err, 1) } targetSignature, err = pwr.ReadSignature(signatureReader) if err != nil { return errors.Wrap(err, 1) } } newPatchRes, newSignatureRes, err := createBothFiles(client, buildID) if err != nil { return errors.Wrap(err, 1) } uploadDone := make(chan bool) uploadErrs := make(chan error) patchWriter, err := uploader.NewResumableUpload(newPatchRes.File.UploadURL, uploadDone, uploadErrs, uploader.ResumableUploadSettings{ Consumer: comm.NewStateConsumer(), }) patchWriter.MaxChunkGroup = *appArgs.maxChunkGroup if err != nil { return errors.Wrap(err, 1) } signatureWriter, err := uploader.NewResumableUpload(newSignatureRes.File.UploadURL, uploadDone, uploadErrs, uploader.ResumableUploadSettings{ Consumer: comm.NewStateConsumer(), }) signatureWriter.MaxChunkGroup = *appArgs.maxChunkGroup if err != nil { return errors.Wrap(err, 1) } comm.Debugf("Launching patch & signature channels") patchCounter := counter.NewWriter(patchWriter) signatureCounter := counter.NewWriter(signatureWriter) // we started walking the source container in the beginning, // we actually need it now. // note that we could actually start diffing before all the file // creation & upload setup is done var sourceContainer *tlc.Container var sourcePool wsync.Pool comm.Debugf("Waiting for source container") select { case walkErr := <-walkErrs: return errors.Wrap(walkErr, 1) case walkies := <-sourceContainerChan: comm.Debugf("Got sourceContainer!") sourceContainer = walkies.container sourcePool = walkies.pool break } comm.Opf("Pushing %s (%s)", humanize.IBytes(uint64(sourceContainer.Size)), sourceContainer.Stats()) comm.Debugf("Building diff context") var readBytes int64 bytesPerSec := float64(0) lastUploadedBytes := int64(0) stopTicking := make(chan struct{}) updateProgress := func() { uploadedBytes := int64(float64(patchWriter.UploadedBytes)) // input bytes that aren't in output, for example: // - bytes that have been compressed away // - bytes that were in old build and were simply reused goneBytes := readBytes - patchWriter.TotalBytes conservativeTotalBytes := sourceContainer.Size - goneBytes leftBytes := conservativeTotalBytes - uploadedBytes if leftBytes > AlmostThereThreshold { netStatus := "- network idle" if bytesPerSec > 1 { netStatus = fmt.Sprintf("@ %s/s", humanize.IBytes(uint64(bytesPerSec))) } comm.ProgressLabel(fmt.Sprintf("%s, %s left", netStatus, humanize.IBytes(uint64(leftBytes)))) } else { comm.ProgressLabel(fmt.Sprintf("- almost there")) } conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes) conservativeProgress = min(1.0, conservativeProgress) comm.Progress(conservativeProgress) comm.ProgressScale(float64(readBytes) / float64(sourceContainer.Size)) } go func() { ticker := time.NewTicker(time.Second * time.Duration(2)) for { select { case <-ticker.C: bytesPerSec = float64(patchWriter.UploadedBytes-lastUploadedBytes) / 2.0 lastUploadedBytes = patchWriter.UploadedBytes updateProgress() case <-stopTicking: break } } }() patchWriter.OnProgress = updateProgress stateConsumer := &state.Consumer{ OnProgress: func(progress float64) { readBytes = int64(float64(sourceContainer.Size) * progress) updateProgress() }, } dctx := &pwr.DiffContext{ Compression: &pwr.CompressionSettings{ Algorithm: pwr.CompressionAlgorithm_BROTLI, Quality: 1, }, SourceContainer: sourceContainer, Pool: sourcePool, TargetContainer: targetSignature.Container, TargetSignature: targetSignature.Hashes, Consumer: stateConsumer, } comm.StartProgress() comm.ProgressScale(0.0) err = dctx.WritePatch(patchCounter, signatureCounter) if err != nil { return errors.Wrap(err, 1) } // close in a goroutine to avoid deadlocking doClose := func(c io.Closer, done chan bool, errs chan error) { closeErr := c.Close() if closeErr != nil { errs <- errors.Wrap(closeErr, 1) return } done <- true } go doClose(patchWriter, uploadDone, uploadErrs) go doClose(signatureWriter, uploadDone, uploadErrs) for c := 0; c < 4; c++ { select { case uploadErr := <-uploadErrs: return errors.Wrap(uploadErr, 1) case <-uploadDone: comm.Debugf("upload done") } } close(stopTicking) comm.ProgressLabel("finalizing build") finalDone := make(chan bool) finalErrs := make(chan error) doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) { _, err = client.FinalizeBuildFile(buildID, fileID, fileSize) if err != nil { errs <- errors.Wrap(err, 1) return } done <- true } go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs) go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs) for i := 0; i < 2; i++ { select { case err := <-finalErrs: return errors.Wrap(err, 1) case <-finalDone: } } comm.EndProgress() { prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count())) percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes) relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size) prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes)) savings := 100.0 - relToNew if dctx.ReusedBytes > 0 { comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize) } else { comm.Statf("Added %s fresh data", prettyFreshSize) } if savings > 0 && !math.IsNaN(savings) { comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew) } else { comm.Statf("%s patch (no savings)", prettyPatchSize) } } comm.Opf("Build is now processing, should be up in a bit (see `butler status`)") comm.Logf("") return nil }
func doFetch(specStr string, outPath string) error { var err error err = os.MkdirAll(outPath, os.FileMode(0755)) if err != nil { return errors.Wrap(err, 1) } outFiles, err := ioutil.ReadDir(outPath) if err != nil { return errors.Wrap(err, 1) } if len(outFiles) > 0 { return fmt.Errorf("Destination directory %s exists and is not empty", outPath) } spec, err := itchio.ParseSpec(specStr) if err != nil { return errors.Wrap(err, 1) } err = spec.EnsureChannel() if err != nil { return errors.Wrap(err, 1) } client, err := authenticateViaOauth() if err != nil { return errors.Wrap(err, 1) } comm.Opf("Getting last build of channel %s", spec.Channel) channelResponse, err := client.GetChannel(spec.Target, spec.Channel) if err != nil { return errors.Wrap(err, 1) } if channelResponse.Channel.Head == nil { return fmt.Errorf("Channel %s doesn't have any builds yet", spec.Channel) } head := *channelResponse.Channel.Head var headArchive *itchio.BuildFileInfo for _, file := range head.Files { comm.Debugf("found file %v", file) if file.Type == itchio.BuildFileType_ARCHIVE && file.SubType == itchio.BuildFileSubType_DEFAULT && file.State == itchio.BuildFileState_UPLOADED { headArchive = file break } } if headArchive == nil { return fmt.Errorf("Channel %s's latest build is still processing", spec.Channel) } dlReader, err := client.DownloadBuildFile(head.ID, headArchive.ID) if err != nil { return errors.Wrap(err, 1) } tmpFile, err := ioutil.TempFile("", "butler-fetch") if err != nil { return errors.Wrap(err, 1) } defer func() { if cErr := os.Remove(tmpFile.Name()); err == nil && cErr != nil { err = cErr } }() comm.Opf("Downloading build %d", head.ID) archiveSize, err := io.Copy(tmpFile, dlReader) if err != nil { return errors.Wrap(err, 1) } _, err = tmpFile.Seek(0, os.SEEK_SET) if err != nil { return errors.Wrap(err, 1) } settings := archiver.ExtractSettings{ Consumer: comm.NewStateConsumer(), } comm.Opf("Extracting into %s", outPath) result, err := archiver.Extract(tmpFile, archiveSize, outPath, settings) if err != nil { return errors.Wrap(err, 1) } comm.Statf("Extracted %d dirs, %d files, %d links into %s", result.Dirs, result.Files, result.Symlinks, outPath) if err != nil { return errors.Wrap(err, 1) } return nil }
func doProbe(patch string) error { patchReader, err := eos.Open(patch) if err != nil { return err } defer patchReader.Close() stats, err := patchReader.Stat() if err != nil { return err } comm.Statf("patch: %s", humanize.IBytes(uint64(stats.Size()))) rctx := wire.NewReadContext(patchReader) err = rctx.ExpectMagic(pwr.PatchMagic) if err != nil { return err } header := &pwr.PatchHeader{} err = rctx.ReadMessage(header) if err != nil { return err } rctx, err = pwr.DecompressWire(rctx, header.Compression) if err != nil { return err } target := &tlc.Container{} err = rctx.ReadMessage(target) if err != nil { return err } source := &tlc.Container{} err = rctx.ReadMessage(source) if err != nil { return err } comm.Statf("target: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats()) comm.Statf("source: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats()) var patchStats []patchStat sh := &pwr.SyncHeader{} rop := &pwr.SyncOp{} for fileIndex, f := range source.Files { stat := patchStat{ fileIndex: int64(fileIndex), freshData: f.Size, } sh.Reset() err = rctx.ReadMessage(sh) if err != nil { return err } if sh.FileIndex != int64(fileIndex) { return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex) } readingOps := true var pos int64 for readingOps { rop.Reset() err = rctx.ReadMessage(rop) if err != nil { return err } switch rop.Type { case pwr.SyncOp_BLOCK_RANGE: fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize lastIndex := rop.BlockIndex + (rop.BlockSpan - 1) lastSize := pwr.ComputeBlockSize(f.Size, lastIndex) totalSize := (fixedSize + lastSize) stat.freshData -= totalSize pos += totalSize case pwr.SyncOp_DATA: totalSize := int64(len(rop.Data)) if *appArgs.verbose { comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)), pos, pos+totalSize) } pos += totalSize case pwr.SyncOp_HEY_YOU_DID_IT: readingOps = false } } patchStats = append(patchStats, stat) } sort.Sort(byDecreasingFreshData(patchStats)) var totalFresh int64 for _, stat := range patchStats { totalFresh += stat.freshData } var eightyFresh = int64(0.8 * float64(totalFresh)) var printedFresh int64 comm.Opf("80%% of fresh data is in the following files:") for _, stat := range patchStats { f := source.Files[stat.fileIndex] comm.Logf("%s in %s (%.2f%% changed)", humanize.IBytes(uint64(stat.freshData)), f.Path, float64(stat.freshData)/float64(f.Size)*100.0) printedFresh += stat.freshData if printedFresh >= eightyFresh { break } } return nil }
func doHeal(dir string, woundsPath string, spec string) error { reader, err := os.Open(woundsPath) if err != nil { return err } defer reader.Close() healer, err := pwr.NewHealer(spec, dir) if err != nil { return err } consumer := comm.NewStateConsumer() healer.SetConsumer(consumer) rctx := wire.NewReadContext(reader) err = rctx.ExpectMagic(pwr.WoundsMagic) if err != nil { return err } wh := &pwr.WoundsHeader{} err = rctx.ReadMessage(wh) if err != nil { return err } container := &tlc.Container{} err = rctx.ReadMessage(container) if err != nil { return err } wounds := make(chan *pwr.Wound) errs := make(chan error) comm.StartProgress() go func() { errs <- healer.Do(container, wounds) }() wound := &pwr.Wound{} for { wound.Reset() err = rctx.ReadMessage(wound) if err != nil { if err == io.EOF { // all good break } } select { case wounds <- wound: // all good case healErr := <-errs: return healErr } } close(wounds) healErr := <-errs comm.EndProgress() if healErr != nil { return healErr } comm.Opf("All healed!") return nil }
func doCmdBsdiff(target string, source string, patch string, concurrency int, measureOverhead bool) error { targetReader, err := os.Open(target) if err != nil { return err } defer targetReader.Close() targetStats, err := targetReader.Stat() if err != nil { return err } sourceReader, err := os.Open(source) if err != nil { return err } defer sourceReader.Close() sourceStats, err := sourceReader.Stat() if err != nil { return err } comm.Opf("Diffing %s (%s) and %s (%s)...", target, humanize.IBytes(uint64(targetStats.Size())), source, humanize.IBytes(uint64(sourceStats.Size()))) patchWriter, err := os.Create(patch) if err != nil { return err } wctx := wire.NewWriteContext(patchWriter) err = wctx.WriteMagic(pwr.PatchMagic) if err != nil { return err } compression := butlerCompressionSettings() err = wctx.WriteMessage(&pwr.PatchHeader{ Compression: &compression, }) if err != nil { return err } wctx, err = pwr.CompressWire(wctx, &compression) if err != nil { return err } targetContainer := &tlc.Container{} targetContainer.Files = []*tlc.File{ &tlc.File{ Path: target, Size: targetStats.Size(), }, } err = wctx.WriteMessage(targetContainer) if err != nil { return err } sourceContainer := &tlc.Container{} sourceContainer.Files = []*tlc.File{ &tlc.File{ Path: source, Size: sourceStats.Size(), }, } err = wctx.WriteMessage(sourceContainer) if err != nil { return err } err = wctx.WriteMessage(&pwr.SyncHeader{ FileIndex: 0, }) if err != nil { return err } err = wctx.WriteMessage(&pwr.SyncOp{ Type: pwr.SyncOp_BSDIFF, FileIndex: 0, }) if err != nil { return err } startTime := time.Now() comm.StartProgress() dc := bsdiff.DiffContext{ MeasureMem: *appArgs.memstats, MeasureParallelOverhead: measureOverhead, SuffixSortConcurrency: concurrency, } err = dc.Do(targetReader, sourceReader, wctx.WriteMessage, comm.NewStateConsumer()) if err != nil { return err } comm.EndProgress() err = wctx.WriteMessage(&pwr.SyncOp{ Type: pwr.SyncOp_HEY_YOU_DID_IT, }) if err != nil { return err } err = wctx.Close() if err != nil { return err } patchStats, err := os.Lstat(patch) if err != nil { return err } duration := time.Since(startTime) perSec := float64(sourceStats.Size()) / duration.Seconds() relToNew := 100.0 * float64(patchStats.Size()) / float64(sourceStats.Size()) comm.Statf("Processed %s @ %s / s, total %s", humanize.IBytes(uint64(sourceStats.Size())), humanize.IBytes(uint64(perSec)), duration) comm.Statf("Wrote %s patch (%.2f%% of total size) to %s", humanize.IBytes(uint64(patchStats.Size())), relToNew, patch) return nil }
func doApply(patch string, target string, output string, inplace bool, signaturePath string, woundsPath string) error { if output == "" { output = target } target = path.Clean(target) output = path.Clean(output) if output == target { if !inplace { comm.Dief("Refusing to destructively patch %s without --inplace", output) } } if signaturePath == "" { comm.Opf("Patching %s", output) } else { comm.Opf("Patching %s with validation", output) } startTime := time.Now() patchReader, err := eos.Open(patch) if err != nil { return errors.Wrap(err, 1) } var signature *pwr.SignatureInfo if signaturePath != "" { sigReader, sigErr := eos.Open(signaturePath) if sigErr != nil { return errors.Wrap(sigErr, 1) } defer sigReader.Close() signature, sigErr = pwr.ReadSignature(sigReader) if sigErr != nil { return errors.Wrap(sigErr, 1) } } actx := &pwr.ApplyContext{ TargetPath: target, OutputPath: output, InPlace: inplace, Signature: signature, WoundsPath: woundsPath, Consumer: comm.NewStateConsumer(), } comm.StartProgress() err = actx.ApplyPatch(patchReader) if err != nil { return errors.Wrap(err, 1) } comm.EndProgress() container := actx.SourceContainer prettySize := humanize.IBytes(uint64(container.Size)) perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds())) if actx.InPlace { statStr := "" if actx.Stats.TouchedFiles > 0 { statStr += fmt.Sprintf("patched %d, ", actx.Stats.TouchedFiles) } if actx.Stats.MovedFiles > 0 { statStr += fmt.Sprintf("renamed %d, ", actx.Stats.MovedFiles) } if actx.Stats.DeletedFiles > 0 { statStr += fmt.Sprintf("deleted %d, ", actx.Stats.DeletedFiles) } comm.Statf("%s (%s stage)", statStr, humanize.IBytes(uint64(actx.Stats.StageSize))) } comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond) if actx.WoundsConsumer != nil && actx.WoundsConsumer.HasWounds() { extra := "" if actx.WoundsPath != "" { extra = fmt.Sprintf(" (written to %s)", actx.WoundsPath) } totalCorrupted := actx.WoundsConsumer.TotalCorrupted() comm.Logf("Result has wounds, %s corrupted data%s", humanize.IBytes(uint64(totalCorrupted)), extra) } return nil }
func doDiff(target string, source string, patch string, compression pwr.CompressionSettings) error { var err error startTime := time.Now() targetSignature := &pwr.SignatureInfo{} targetSignature.Container, err = tlc.WalkAny(target, filterPaths) if err != nil { // Signature file perhaps? var signatureReader io.ReadCloser signatureReader, err = eos.Open(target) if err != nil { return errors.Wrap(err, 1) } targetSignature, err = pwr.ReadSignature(signatureReader) if err != nil { if errors.Is(err, wire.ErrFormat) { return fmt.Errorf("unrecognized target %s (not a container, not a signature file)", target) } return errors.Wrap(err, 1) } comm.Opf("Read signature from %s", target) err = signatureReader.Close() if err != nil { return errors.Wrap(err, 1) } } else { // Container (dir, archive, etc.) comm.Opf("Hashing %s", target) comm.StartProgress() var targetPool wsync.Pool targetPool, err = pools.New(targetSignature.Container, target) if err != nil { return errors.Wrap(err, 1) } targetSignature.Hashes, err = pwr.ComputeSignature(targetSignature.Container, targetPool, comm.NewStateConsumer()) comm.EndProgress() if err != nil { return errors.Wrap(err, 1) } { prettySize := humanize.IBytes(uint64(targetSignature.Container.Size)) perSecond := humanize.IBytes(uint64(float64(targetSignature.Container.Size) / time.Since(startTime).Seconds())) comm.Statf("%s (%s) @ %s/s\n", prettySize, targetSignature.Container.Stats(), perSecond) } } startTime = time.Now() var sourceContainer *tlc.Container sourceContainer, err = tlc.WalkAny(source, filterPaths) if err != nil { return errors.Wrap(err, 1) } var sourcePool wsync.Pool sourcePool, err = pools.New(sourceContainer, source) if err != nil { return errors.Wrap(err, 1) } patchWriter, err := os.Create(patch) if err != nil { return errors.Wrap(err, 1) } defer patchWriter.Close() signaturePath := patch + ".sig" signatureWriter, err := os.Create(signaturePath) if err != nil { return errors.Wrap(err, 1) } defer signatureWriter.Close() patchCounter := counter.NewWriter(patchWriter) signatureCounter := counter.NewWriter(signatureWriter) dctx := &pwr.DiffContext{ SourceContainer: sourceContainer, Pool: sourcePool, TargetContainer: targetSignature.Container, TargetSignature: targetSignature.Hashes, Consumer: comm.NewStateConsumer(), Compression: &compression, } comm.Opf("Diffing %s", source) comm.StartProgress() err = dctx.WritePatch(patchCounter, signatureCounter) if err != nil { return errors.Wrap(err, 1) } comm.EndProgress() totalDuration := time.Since(startTime) { prettySize := humanize.IBytes(uint64(sourceContainer.Size)) perSecond := humanize.IBytes(uint64(float64(sourceContainer.Size) / totalDuration.Seconds())) comm.Statf("%s (%s) @ %s/s\n", prettySize, sourceContainer.Stats(), perSecond) } if *diffArgs.verify { tmpDir, err := ioutil.TempDir("", "pwr") if err != nil { return errors.Wrap(err, 1) } defer os.RemoveAll(tmpDir) apply(patch, target, tmpDir, false, signaturePath, "") } { prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count())) percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes) relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size) prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes)) comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize) comm.Statf("%s patch (%.2f%% of the full size) in %s", prettyPatchSize, relToNew, totalDuration) } return nil }
func applyUpgrade(before string, after string) error { execPath, err := osext.Executable() if err != nil { return err } oldPath := execPath + ".old" newPath := execPath + ".new" gzPath := newPath + ".gz" err = os.RemoveAll(newPath) if err != nil { return err } err = os.RemoveAll(gzPath) if err != nil { return err } ext := "" if runtime.GOOS == "windows" { ext = ".exe" } fragment := fmt.Sprintf("v%s", after) if after == "head" { fragment = "head" } execURL := fmt.Sprintf("%s/%s/butler%s", updateBaseURL, fragment, ext) gzURL := fmt.Sprintf("%s/%s/butler.gz", updateBaseURL, fragment) comm.Opf("%s", gzURL) err = func() error { _, gErr := tryDl(gzURL, gzPath) if gErr != nil { return gErr } fr, gErr := os.Open(gzPath) if gErr != nil { return gErr } defer fr.Close() gr, gErr := gzip.NewReader(fr) if gErr != nil { return gErr } fw, gErr := os.Create(newPath) if gErr != nil { return gErr } defer fw.Close() _, gErr = io.Copy(fw, gr) if gErr != nil { return gErr } return nil }() if err != nil { comm.Opf("Falling back to %s", execURL) _, err = tryDl(execURL, newPath) must(err) } err = os.Chmod(newPath, os.FileMode(0755)) if err != nil { return err } comm.Opf("Backing up current version to %s just in case...", oldPath) err = os.Rename(execPath, oldPath) if err != nil { return err } err = os.Rename(newPath, execPath) if err != nil { return err } err = os.Remove(oldPath) if err != nil { if os.IsPermission(err) && runtime.GOOS == "windows" { // poor windows doesn't like us removing executables from under it // I vote we move on and let butler.exe.old hang around. } else { return err } } comm.Statf("Upgraded butler from %s to %s. Have a nice day!", before, after) return nil }