func doLogin() error {
var identity = *appArgs.identity
_, err := os.Lstat(identity)
hasSavedCredentials := !os.IsNotExist(err)
if hasSavedCredentials {
client, err := authenticateViaOauth()
if err != nil {
return errors.Wrap(err, 1)
}
_, err = client.WharfStatus()
if err != nil {
return errors.Wrap(err, 1)
}
comm.Logf("Your local credentials are valid!\n")
comm.Logf("If you want to log in as another account, use the `butler logout` command first, or specify a different credentials path with the `-i` flag.")
} else {
// this does the full login flow + saves
_, err := authenticateViaOauth()
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}
return nil
}
func which() {
p, err := osext.Executable()
must(err)
comm.Logf("You're running butler %s, from the following path:", versionString)
comm.Logf("%s", p)
}
func checkHashes(header http.Header, file string) (bool, error) {
googHashes := header[http.CanonicalHeaderKey("x-goog-hash")]
for _, googHash := range googHashes {
tokens := strings.SplitN(googHash, "=", 2)
hashType := tokens[0]
hashValue, err := base64.StdEncoding.DecodeString(tokens[1])
if err != nil {
comm.Logf("Could not verify %s hash: %s", hashType, err)
continue
}
start := time.Now()
checked, err := checkHash(hashType, hashValue, file)
if err != nil {
return false, errors.Wrap(err, 1)
}
if checked {
comm.Debugf("%10s pass (took %s)", hashType, time.Since(start))
} else {
comm.Debugf("%10s skip (use --thorough to force check)", hashType)
}
}
return true, nil
}
func setupHTTPDebug() {
debugPort := os.Getenv("BUTLER_DEBUG_PORT")
if debugPort == "" {
return
}
addr := fmt.Sprintf("localhost:%s", debugPort)
go func() {
err := http.ListenAndServe(addr, nil)
if err != nil {
comm.Logf("http debug error: %s", err.Error())
}
}()
comm.Logf("serving pprof debug interface on %s", addr)
}
func doLogout() error {
var identity = *appArgs.identity
_, err := os.Lstat(identity)
if err != nil {
if os.IsNotExist(err) {
fmt.Println("No saved credentials at", identity)
fmt.Println("Nothing to do.")
return nil
}
}
comm.Notice("Important note", []string{
"Note: this command will not invalidate the API key itself.",
"If you wish to revoke it (for example, because it's been compromised), you should do so in your user settings:",
"",
fmt.Sprintf(" %s/user/settings\n\n", *appArgs.address),
})
comm.Logf("")
if !comm.YesNo("Do you want to erase your saved API key?") {
fmt.Println("Okay, not erasing credentials. Bye!")
return nil
}
err = os.Remove(identity)
if err != nil {
return errors.Wrap(err, 1)
}
fmt.Println("You've successfully erased the API key that was saved on your computer.")
return nil
}
func untar(file string, dir string) {
settings := archiver.ExtractSettings{
Consumer: comm.NewStateConsumer(),
}
comm.StartProgress()
res, err := archiver.ExtractTar(file, dir, settings)
comm.EndProgress()
must(err)
comm.Logf("Extracted %d dirs, %d files, %d symlinks", res.Dirs, res.Files, res.Symlinks)
}
func doUpgrade(head bool) error {
if head {
if !comm.YesNo("Do you want to upgrade to the bleeding-edge version? Things may break!") {
comm.Logf("Okay, not upgrading. Bye!")
return nil
}
return applyUpgrade("head", "head")
}
if version == "head" {
comm.Statf("Bleeding-edge, not upgrading unless told to.")
comm.Logf("(Use `--head` if you want to upgrade to the latest bleeding-edge version)")
return nil
}
comm.Opf("Looking for upgrades...")
currentVer, latestVer, err := queryLatestVersion()
if err != nil {
return fmt.Errorf("Version check failed: %s", err.Error())
}
if latestVer == nil || currentVer.GTE(*latestVer) {
comm.Statf("Your butler is up-to-date. Have a nice day!")
return nil
}
comm.Statf("Current version: %s", currentVer.String())
comm.Statf("Latest version : %s", latestVer.String())
if !comm.YesNo("Do you want to upgrade now?") {
comm.Logf("Okay, not upgrading. Bye!")
return nil
}
must(applyUpgrade(currentVer.String(), latestVer.String()))
return nil
}
func doStatus(specStr string) error {
spec, err := itchio.ParseSpec(specStr)
if err != nil {
return errors.Wrap(err, 1)
}
client, err := authenticateViaOauth()
if err != nil {
return errors.Wrap(err, 1)
}
listChannelsResp, err := client.ListChannels(spec.Target)
if err != nil {
return errors.Wrap(err, 1)
}
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Channel", "Upload", "Build", "State"})
found := false
for _, ch := range listChannelsResp.Channels {
if spec.Channel != "" && ch.Name != spec.Channel {
continue
}
found = true
if ch.Head != nil {
files := ch.Head.Files
line := []string{ch.Name, fmt.Sprintf("#%d", ch.Upload.ID), buildState(ch.Head), filesState(files)}
table.Append(line)
} else {
line := []string{ch.Name, fmt.Sprintf("#%d", ch.Upload.ID), "No builds yet"}
table.Append(line)
}
if ch.Pending != nil {
files := ch.Pending.Files
line := []string{"", "", buildState(ch.Pending), filesState(files)}
table.Append(line)
}
}
if found {
table.Render()
} else {
comm.Logf("No channel %s found for %s", spec.Channel, spec.Target)
}
return nil
}
func sizeof(path string) {
totalSize := int64(0)
inc := func(_ string, f os.FileInfo, err error) error {
if err != nil {
return nil
}
totalSize += f.Size()
return nil
}
filepath.Walk(path, inc)
comm.Logf("Total size of %s: %s", path, humanize.IBytes(uint64(totalSize)))
comm.Result(totalSize)
}
func wipe(path string) {
// why have retry logic built into wipe? sometimes when uninstalling
// games on windows, the os will randomly return I/O errors, retrying
// usually helps.
attempt := 0
sleepPatterns := []time.Duration{
time.Millisecond * 200,
time.Millisecond * 800,
time.Millisecond * 1600,
time.Millisecond * 3200,
}
for attempt <= len(sleepPatterns) {
err := tryWipe(path)
if err == nil {
break
}
if attempt == len(sleepPatterns) {
comm.Dief("Could not wipe %s: %s", path, err.Error())
}
comm.Logf("While wiping %s: %s", path, err.Error())
comm.Logf("Trying to brute-force permissions, who knows...")
err = tryChmod(path)
if err != nil {
comm.Logf("While bruteforcing: %s", err.Error())
}
comm.Logf("Sleeping for a bit before we retry...")
sleepDuration := sleepPatterns[attempt]
time.Sleep(sleepDuration)
attempt++
}
}
func unzip(file string, dir string, resumeFile string) {
comm.Opf("Extracting zip %s to %s", file, dir)
settings := archiver.ExtractSettings{
Consumer: comm.NewStateConsumer(),
ResumeFrom: resumeFile,
OnUncompressedSizeKnown: func(uncompressedSize int64) {
comm.StartProgressWithTotalBytes(uncompressedSize)
},
}
res, err := archiver.ExtractPath(file, dir, settings)
comm.EndProgress()
must(err)
comm.Logf("Extracted %d dirs, %d files, %d symlinks", res.Dirs, res.Files, res.Symlinks)
}
func versionCheck() {
currentVer, latestVer, err := queryLatestVersion()
if err != nil {
comm.Logf("Version check failed: %s", err.Error())
}
if currentVer == nil || latestVer == nil {
return
}
if latestVer.GT(*currentVer) {
comm.Notice("New version available",
[]string{
fmt.Sprintf("Current version: %s", version),
fmt.Sprintf("Latest version: %s", latestVer),
"",
"Run `butler upgrade` to get it.",
})
}
}
func doProbe(patch string) error {
patchReader, err := eos.Open(patch)
if err != nil {
return err
}
defer patchReader.Close()
stats, err := patchReader.Stat()
if err != nil {
return err
}
comm.Statf("patch: %s", humanize.IBytes(uint64(stats.Size())))
rctx := wire.NewReadContext(patchReader)
err = rctx.ExpectMagic(pwr.PatchMagic)
if err != nil {
return err
}
header := &pwr.PatchHeader{}
err = rctx.ReadMessage(header)
if err != nil {
return err
}
rctx, err = pwr.DecompressWire(rctx, header.Compression)
if err != nil {
return err
}
target := &tlc.Container{}
err = rctx.ReadMessage(target)
if err != nil {
return err
}
source := &tlc.Container{}
err = rctx.ReadMessage(source)
if err != nil {
return err
}
comm.Statf("target: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats())
comm.Statf("source: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats())
var patchStats []patchStat
sh := &pwr.SyncHeader{}
rop := &pwr.SyncOp{}
for fileIndex, f := range source.Files {
stat := patchStat{
fileIndex: int64(fileIndex),
freshData: f.Size,
}
sh.Reset()
err = rctx.ReadMessage(sh)
if err != nil {
return err
}
if sh.FileIndex != int64(fileIndex) {
return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex)
}
readingOps := true
var pos int64
for readingOps {
rop.Reset()
err = rctx.ReadMessage(rop)
if err != nil {
return err
}
switch rop.Type {
case pwr.SyncOp_BLOCK_RANGE:
fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize
lastIndex := rop.BlockIndex + (rop.BlockSpan - 1)
lastSize := pwr.ComputeBlockSize(f.Size, lastIndex)
totalSize := (fixedSize + lastSize)
stat.freshData -= totalSize
pos += totalSize
case pwr.SyncOp_DATA:
totalSize := int64(len(rop.Data))
if *appArgs.verbose {
comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)),
pos, pos+totalSize)
}
pos += totalSize
case pwr.SyncOp_HEY_YOU_DID_IT:
readingOps = false
}
}
patchStats = append(patchStats, stat)
}
sort.Sort(byDecreasingFreshData(patchStats))
var totalFresh int64
for _, stat := range patchStats {
totalFresh += stat.freshData
}
var eightyFresh = int64(0.8 * float64(totalFresh))
var printedFresh int64
comm.Opf("80%% of fresh data is in the following files:")
for _, stat := range patchStats {
f := source.Files[stat.fileIndex]
comm.Logf("%s in %s (%.2f%% changed)",
humanize.IBytes(uint64(stat.freshData)),
f.Path,
float64(stat.freshData)/float64(f.Size)*100.0)
printedFresh += stat.freshData
if printedFresh >= eightyFresh {
break
}
}
return nil
}
func doCp(srcPath string, destPath string, resume bool) error {
src, err := eos.Open(srcPath)
if err != nil {
return err
}
defer src.Close()
dir := filepath.Dir(destPath)
err = os.MkdirAll(dir, 0755)
if err != nil {
return err
}
flags := os.O_CREATE | os.O_WRONLY
dest, err := os.OpenFile(destPath, flags, 0644)
if err != nil {
return err
}
defer dest.Close()
stats, err := src.Stat()
if err != nil {
return err
}
totalBytes := int64(stats.Size())
startOffset := int64(0)
if resume {
startOffset, err = dest.Seek(0, os.SEEK_END)
if err != nil {
return err
}
if startOffset == 0 {
comm.Logf("Downloading %s", humanize.IBytes(uint64(totalBytes)))
} else if startOffset > totalBytes {
comm.Logf("Existing data too big (%s > %s), starting over", humanize.IBytes(uint64(startOffset)), humanize.IBytes(uint64(totalBytes)))
} else if startOffset == totalBytes {
comm.Logf("All %s already there", humanize.IBytes(uint64(totalBytes)))
return nil
}
comm.Logf("Resuming at %s / %s", humanize.IBytes(uint64(startOffset)), humanize.IBytes(uint64(totalBytes)))
_, err = src.Seek(startOffset, os.SEEK_SET)
if err != nil {
return err
}
} else {
comm.Logf("Downloading %s", humanize.IBytes(uint64(totalBytes)))
}
start := time.Now()
comm.Progress(float64(startOffset) / float64(totalBytes))
comm.StartProgressWithTotalBytes(totalBytes)
cw := counter.NewWriterCallback(func(count int64) {
alpha := float64(startOffset+count) / float64(totalBytes)
comm.Progress(alpha)
}, dest)
copiedBytes, err := io.Copy(cw, src)
if err != nil {
return err
}
comm.EndProgress()
totalDuration := time.Since(start)
prettyStartOffset := humanize.IBytes(uint64(startOffset))
prettySize := humanize.IBytes(uint64(copiedBytes))
perSecond := humanize.IBytes(uint64(float64(totalBytes-startOffset) / totalDuration.Seconds()))
comm.Statf("%s + %s copied @ %s/s\n", prettyStartOffset, prettySize, perSecond)
return nil
}
func tryDl(url string, dest string) (int64, error) {
existingBytes := int64(0)
stats, err := os.Lstat(dest)
if err == nil {
existingBytes = stats.Size()
}
client := timeout.NewDefaultClient()
req, _ := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", userAgent())
byteRange := fmt.Sprintf("bytes=%d-", existingBytes)
req.Header.Set("Range", byteRange)
resp, err := client.Do(req)
if err != nil {
return 0, errors.Wrap(err, 1)
}
defer resp.Body.Close()
doDownload := true
totalBytes := existingBytes + resp.ContentLength
hostInfo := fmt.Sprintf("%s at %s", resp.Header.Get("Server"), req.Host)
switch resp.StatusCode {
case 200: // OK
comm.Debugf("HTTP 200 OK (no byte range support)")
totalBytes = resp.ContentLength
if existingBytes == resp.ContentLength {
// already have the exact same number of bytes, hopefully the same ones
doDownload = false
} else {
// will send data, but doesn't support byte ranges
existingBytes = 0
os.Truncate(dest, 0)
}
case 206: // Partial Content
comm.Debugf("HTTP 206 Partial Content")
// will send incremental data
case 416: // Requested Range not Satisfiable
comm.Debugf("HTTP 416 Requested Range not Satisfiable")
// already has everything
doDownload = false
req, _ := http.NewRequest("HEAD", url, nil)
req.Header.Set("User-Agent", userAgent())
resp, err = client.Do(req)
if err != nil {
return 0, errors.Wrap(err, 1)
}
if existingBytes > resp.ContentLength {
comm.Debugf("Existing file too big (%d), truncating to %d", existingBytes, resp.ContentLength)
existingBytes = resp.ContentLength
os.Truncate(dest, existingBytes)
}
totalBytes = existingBytes
default:
return 0, fmt.Errorf("%s responded with HTTP %s", hostInfo, resp.Status)
}
if doDownload {
if existingBytes > 0 {
comm.Logf("Resuming (%s + %s = %s) download from %s", humanize.IBytes(uint64(existingBytes)), humanize.IBytes(uint64(resp.ContentLength)), humanize.IBytes(uint64(totalBytes)), hostInfo)
} else {
comm.Logf("Downloading %s from %s", humanize.IBytes(uint64(resp.ContentLength)), hostInfo)
}
err = appendAllToFile(resp.Body, dest, existingBytes, totalBytes)
if err != nil {
return 0, errors.Wrap(err, 1)
}
} else {
comm.Log("Already fully downloaded")
}
_, err = checkIntegrity(resp, totalBytes, dest)
if err != nil {
comm.Log("Integrity checks failed, truncating")
os.Truncate(dest, 0)
return 0, errors.Wrap(err, 1)
}
return totalBytes, nil
}
func file(path string) {
reader, err := eos.Open(path)
must(err)
defer reader.Close()
stats, err := reader.Stat()
if os.IsNotExist(err) {
comm.Dief("%s: no such file or directory", path)
}
must(err)
if stats.IsDir() {
comm.Logf("%s: directory", path)
return
}
if stats.Size() == 0 {
comm.Logf("%s: empty file. peaceful.", path)
return
}
prettySize := humanize.IBytes(uint64(stats.Size()))
var magic int32
must(binary.Read(reader, wire.Endianness, &magic))
switch magic {
case pwr.PatchMagic:
{
ph := &pwr.PatchHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(ph))
rctx, err = pwr.DecompressWire(rctx, ph.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container)) // target container
container.Reset()
must(rctx.ReadMessage(container)) // source container
comm.Logf("%s: %s wharf patch file (%s) with %s", path, prettySize, ph.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/patch",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.SignatureMagic:
{
sh := &pwr.SignatureHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(sh))
rctx, err = pwr.DecompressWire(rctx, sh.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
comm.Logf("%s: %s wharf signature file (%s) with %s", path, prettySize, sh.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/signature",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.ManifestMagic:
{
mh := &pwr.ManifestHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(mh))
rctx, err = pwr.DecompressWire(rctx, mh.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
comm.Logf("%s: %s wharf manifest file (%s) with %s", path, prettySize, mh.GetCompression().ToString(), container.Stats())
comm.Result(ContainerResult{
Type: "wharf/manifest",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
}
case pwr.WoundsMagic:
{
wh := &pwr.WoundsHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(wh))
container := &tlc.Container{}
must(rctx.ReadMessage(container))
files := make(map[int64]bool)
totalWounds := int64(0)
for {
wound := &pwr.Wound{}
err = rctx.ReadMessage(wound)
if err != nil {
if errors.Is(err, io.EOF) {
break
} else {
must(err)
}
}
if wound.Kind == pwr.WoundKind_FILE {
totalWounds += (wound.End - wound.Start)
files[wound.Index] = true
}
}
comm.Logf("%s: %s wharf wounds file with %s, %s wounds in %d files", path, prettySize, container.Stats(),
humanize.IBytes(uint64(totalWounds)), len(files))
comm.Result(ContainerResult{
Type: "wharf/wounds",
})
}
default:
_, err := reader.Seek(0, os.SEEK_SET)
must(err)
wasZip := func() bool {
zr, err := zip.NewReader(reader, stats.Size())
if err != nil {
if err != zip.ErrFormat {
must(err)
}
return false
}
container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
must(err)
comm.Logf("%s: %s zip file with %s", path, prettySize, container.Stats())
comm.Result(ContainerResult{
Type: "zip",
NumFiles: len(container.Files),
NumDirs: len(container.Dirs),
NumSymlinks: len(container.Symlinks),
UncompressedSize: container.Size,
})
return true
}()
if !wasZip {
comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
}
}
}
func doApply(patch string, target string, output string, inplace bool, signaturePath string, woundsPath string) error {
if output == "" {
output = target
}
target = path.Clean(target)
output = path.Clean(output)
if output == target {
if !inplace {
comm.Dief("Refusing to destructively patch %s without --inplace", output)
}
}
if signaturePath == "" {
comm.Opf("Patching %s", output)
} else {
comm.Opf("Patching %s with validation", output)
}
startTime := time.Now()
patchReader, err := eos.Open(patch)
if err != nil {
return errors.Wrap(err, 1)
}
var signature *pwr.SignatureInfo
if signaturePath != "" {
sigReader, sigErr := eos.Open(signaturePath)
if sigErr != nil {
return errors.Wrap(sigErr, 1)
}
defer sigReader.Close()
signature, sigErr = pwr.ReadSignature(sigReader)
if sigErr != nil {
return errors.Wrap(sigErr, 1)
}
}
actx := &pwr.ApplyContext{
TargetPath: target,
OutputPath: output,
InPlace: inplace,
Signature: signature,
WoundsPath: woundsPath,
Consumer: comm.NewStateConsumer(),
}
comm.StartProgress()
err = actx.ApplyPatch(patchReader)
if err != nil {
return errors.Wrap(err, 1)
}
comm.EndProgress()
container := actx.SourceContainer
prettySize := humanize.IBytes(uint64(container.Size))
perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
if actx.InPlace {
statStr := ""
if actx.Stats.TouchedFiles > 0 {
statStr += fmt.Sprintf("patched %d, ", actx.Stats.TouchedFiles)
}
if actx.Stats.MovedFiles > 0 {
statStr += fmt.Sprintf("renamed %d, ", actx.Stats.MovedFiles)
}
if actx.Stats.DeletedFiles > 0 {
statStr += fmt.Sprintf("deleted %d, ", actx.Stats.DeletedFiles)
}
comm.Statf("%s (%s stage)", statStr, humanize.IBytes(uint64(actx.Stats.StageSize)))
}
comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond)
if actx.WoundsConsumer != nil && actx.WoundsConsumer.HasWounds() {
extra := ""
if actx.WoundsPath != "" {
extra = fmt.Sprintf(" (written to %s)", actx.WoundsPath)
}
totalCorrupted := actx.WoundsConsumer.TotalCorrupted()
comm.Logf("Result has wounds, %s corrupted data%s", humanize.IBytes(uint64(totalCorrupted)), extra)
}
return nil
}
// Does not preserve users, nor permission, except the executable bit
func ditto(src string, dst string) {
comm.Debugf("rsync -a %s %s", src, dst)
totalSize := int64(0)
doneSize := int64(0)
oldProgress := 0.0
inc := func(_ string, f os.FileInfo, err error) error {
if err != nil {
return nil
}
totalSize += f.Size()
return nil
}
onFile := func(path string, f os.FileInfo, err error) error {
if err != nil {
comm.Logf("ignoring error %s", err.Error())
return nil
}
rel, err := filepath.Rel(src, path)
must(err)
dstpath := filepath.Join(dst, rel)
mode := f.Mode()
switch {
case mode.IsDir():
dittoMkdir(dstpath)
case mode.IsRegular():
dittoReg(path, dstpath, os.FileMode(f.Mode()&archiver.LuckyMode|archiver.ModeMask))
case (mode&os.ModeSymlink > 0):
dittoSymlink(path, dstpath, f)
}
comm.Debug(rel)
doneSize += f.Size()
progress := float64(doneSize) / float64(totalSize)
if progress-oldProgress > 0.01 {
oldProgress = progress
comm.Progress(progress)
}
return nil
}
rootinfo, err := os.Lstat(src)
must(err)
if rootinfo.IsDir() {
totalSize = 0
comm.Logf("Counting files in %s...", src)
filepath.Walk(src, inc)
comm.Logf("Mirroring...")
filepath.Walk(src, onFile)
} else {
totalSize = rootinfo.Size()
onFile(src, rootinfo, nil)
}
comm.EndProgress()
}
func doPush(buildPath string, specStr string, userVersion string, fixPerms bool) error {
// start walking source container while waiting on auth flow
sourceContainerChan := make(chan walkResult)
walkErrs := make(chan error)
go doWalk(buildPath, sourceContainerChan, walkErrs, fixPerms)
spec, err := itchio.ParseSpec(specStr)
if err != nil {
return errors.Wrap(err, 1)
}
err = spec.EnsureChannel()
if err != nil {
return errors.Wrap(err, 1)
}
client, err := authenticateViaOauth()
if err != nil {
return errors.Wrap(err, 1)
}
newBuildRes, err := client.CreateBuild(spec.Target, spec.Channel, userVersion)
if err != nil {
return errors.Wrap(err, 1)
}
buildID := newBuildRes.Build.ID
parentID := newBuildRes.Build.ParentBuild.ID
var targetSignature *pwr.SignatureInfo
if parentID == 0 {
comm.Opf("For channel `%s`: pushing first build", spec.Channel)
targetSignature = &pwr.SignatureInfo{
Container: &tlc.Container{},
Hashes: make([]wsync.BlockHash, 0),
}
} else {
comm.Opf("For channel `%s`: last build is %d, downloading its signature", spec.Channel, parentID)
var buildFiles itchio.ListBuildFilesResponse
buildFiles, err = client.ListBuildFiles(parentID)
if err != nil {
return errors.Wrap(err, 1)
}
signatureFile := itchio.FindBuildFile(itchio.BuildFileType_SIGNATURE, buildFiles.Files)
if signatureFile == nil {
comm.Dief("Could not find signature for parent build %d, aborting", parentID)
}
var signatureReader io.Reader
signatureReader, err = client.DownloadBuildFile(parentID, signatureFile.ID)
if err != nil {
return errors.Wrap(err, 1)
}
targetSignature, err = pwr.ReadSignature(signatureReader)
if err != nil {
return errors.Wrap(err, 1)
}
}
newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)
if err != nil {
return errors.Wrap(err, 1)
}
uploadDone := make(chan bool)
uploadErrs := make(chan error)
patchWriter, err := uploader.NewResumableUpload(newPatchRes.File.UploadURL,
uploadDone, uploadErrs, uploader.ResumableUploadSettings{
Consumer: comm.NewStateConsumer(),
})
patchWriter.MaxChunkGroup = *appArgs.maxChunkGroup
if err != nil {
return errors.Wrap(err, 1)
}
signatureWriter, err := uploader.NewResumableUpload(newSignatureRes.File.UploadURL,
uploadDone, uploadErrs, uploader.ResumableUploadSettings{
Consumer: comm.NewStateConsumer(),
})
signatureWriter.MaxChunkGroup = *appArgs.maxChunkGroup
if err != nil {
return errors.Wrap(err, 1)
}
comm.Debugf("Launching patch & signature channels")
patchCounter := counter.NewWriter(patchWriter)
signatureCounter := counter.NewWriter(signatureWriter)
// we started walking the source container in the beginning,
// we actually need it now.
// note that we could actually start diffing before all the file
// creation & upload setup is done
var sourceContainer *tlc.Container
var sourcePool wsync.Pool
comm.Debugf("Waiting for source container")
select {
case walkErr := <-walkErrs:
return errors.Wrap(walkErr, 1)
case walkies := <-sourceContainerChan:
comm.Debugf("Got sourceContainer!")
sourceContainer = walkies.container
sourcePool = walkies.pool
break
}
comm.Opf("Pushing %s (%s)", humanize.IBytes(uint64(sourceContainer.Size)), sourceContainer.Stats())
comm.Debugf("Building diff context")
var readBytes int64
bytesPerSec := float64(0)
lastUploadedBytes := int64(0)
stopTicking := make(chan struct{})
updateProgress := func() {
uploadedBytes := int64(float64(patchWriter.UploadedBytes))
// input bytes that aren't in output, for example:
// - bytes that have been compressed away
// - bytes that were in old build and were simply reused
goneBytes := readBytes - patchWriter.TotalBytes
conservativeTotalBytes := sourceContainer.Size - goneBytes
leftBytes := conservativeTotalBytes - uploadedBytes
if leftBytes > AlmostThereThreshold {
netStatus := "- network idle"
if bytesPerSec > 1 {
netStatus = fmt.Sprintf("@ %s/s", humanize.IBytes(uint64(bytesPerSec)))
}
comm.ProgressLabel(fmt.Sprintf("%s, %s left", netStatus, humanize.IBytes(uint64(leftBytes))))
} else {
comm.ProgressLabel(fmt.Sprintf("- almost there"))
}
conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
conservativeProgress = min(1.0, conservativeProgress)
comm.Progress(conservativeProgress)
comm.ProgressScale(float64(readBytes) / float64(sourceContainer.Size))
}
go func() {
ticker := time.NewTicker(time.Second * time.Duration(2))
for {
select {
case <-ticker.C:
bytesPerSec = float64(patchWriter.UploadedBytes-lastUploadedBytes) / 2.0
lastUploadedBytes = patchWriter.UploadedBytes
updateProgress()
case <-stopTicking:
break
}
}
}()
patchWriter.OnProgress = updateProgress
stateConsumer := &state.Consumer{
OnProgress: func(progress float64) {
readBytes = int64(float64(sourceContainer.Size) * progress)
updateProgress()
},
}
dctx := &pwr.DiffContext{
Compression: &pwr.CompressionSettings{
Algorithm: pwr.CompressionAlgorithm_BROTLI,
Quality: 1,
},
SourceContainer: sourceContainer,
Pool: sourcePool,
TargetContainer: targetSignature.Container,
TargetSignature: targetSignature.Hashes,
Consumer: stateConsumer,
}
comm.StartProgress()
comm.ProgressScale(0.0)
err = dctx.WritePatch(patchCounter, signatureCounter)
if err != nil {
return errors.Wrap(err, 1)
}
// close in a goroutine to avoid deadlocking
doClose := func(c io.Closer, done chan bool, errs chan error) {
closeErr := c.Close()
if closeErr != nil {
errs <- errors.Wrap(closeErr, 1)
return
}
done <- true
}
go doClose(patchWriter, uploadDone, uploadErrs)
go doClose(signatureWriter, uploadDone, uploadErrs)
for c := 0; c < 4; c++ {
select {
case uploadErr := <-uploadErrs:
return errors.Wrap(uploadErr, 1)
case <-uploadDone:
comm.Debugf("upload done")
}
}
close(stopTicking)
comm.ProgressLabel("finalizing build")
finalDone := make(chan bool)
finalErrs := make(chan error)
doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
if err != nil {
errs <- errors.Wrap(err, 1)
return
}
done <- true
}
go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)
for i := 0; i < 2; i++ {
select {
case err := <-finalErrs:
return errors.Wrap(err, 1)
case <-finalDone:
}
}
comm.EndProgress()
{
prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count()))
percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes))
savings := 100.0 - relToNew
if dctx.ReusedBytes > 0 {
comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
} else {
comm.Statf("Added %s fresh data", prettyFreshSize)
}
if savings > 0 && !math.IsNaN(savings) {
comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
} else {
comm.Statf("%s patch (no savings)", prettyPatchSize)
}
}
comm.Opf("Build is now processing, should be up in a bit (see `butler status`)")
comm.Logf("")
return nil
}
func ls(path string) {
reader, err := eos.Open(path)
must(err)
defer reader.Close()
stats, err := reader.Stat()
if os.IsNotExist(err) {
comm.Dief("%s: no such file or directory", path)
}
must(err)
if stats.IsDir() {
comm.Logf("%s: directory", path)
return
}
if stats.Size() == 0 {
comm.Logf("%s: empty file. peaceful.", path)
return
}
log := func(line string) {
comm.Logf(line)
}
var magic int32
must(binary.Read(reader, wire.Endianness, &magic))
switch magic {
case pwr.PatchMagic:
{
h := &pwr.PatchHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
log("pre-patch container:")
container.Print(log)
container.Reset()
must(rctx.ReadMessage(container))
log("================================")
log("post-patch container:")
container.Print(log)
}
case pwr.SignatureMagic:
{
h := &pwr.SignatureHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
}
case pwr.ManifestMagic:
{
h := &pwr.ManifestHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(h))
rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
must(err)
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
}
case pwr.WoundsMagic:
{
wh := &pwr.WoundsHeader{}
rctx := wire.NewReadContext(reader)
must(rctx.ReadMessage(wh))
container := &tlc.Container{}
must(rctx.ReadMessage(container))
container.Print(log)
for {
wound := &pwr.Wound{}
err = rctx.ReadMessage(wound)
if err != nil {
if errors.Is(err, io.EOF) {
break
} else {
must(err)
}
}
comm.Logf(wound.PrettyString(container))
}
}
default:
_, err := reader.Seek(0, os.SEEK_SET)
must(err)
wasZip := func() bool {
zr, err := zip.NewReader(reader, stats.Size())
if err != nil {
if err != zip.ErrFormat {
must(err)
}
return false
}
container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
must(err)
container.Print(log)
return true
}()
if !wasZip {
comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
}
}
}