Esempio n. 1
0
func doLogin() error {
	var identity = *appArgs.identity
	_, err := os.Lstat(identity)
	hasSavedCredentials := !os.IsNotExist(err)

	if hasSavedCredentials {
		client, err := authenticateViaOauth()
		if err != nil {
			return errors.Wrap(err, 1)
		}

		_, err = client.WharfStatus()
		if err != nil {
			return errors.Wrap(err, 1)
		}

		comm.Logf("Your local credentials are valid!\n")
		comm.Logf("If you want to log in as another account, use the `butler logout` command first, or specify a different credentials path with the `-i` flag.")
	} else {
		// this does the full login flow + saves
		_, err := authenticateViaOauth()
		if err != nil {
			return errors.Wrap(err, 1)
		}
		return nil
	}

	return nil
}
Esempio n. 2
0
func which() {
	p, err := osext.Executable()
	must(err)

	comm.Logf("You're running butler %s, from the following path:", versionString)
	comm.Logf("%s", p)
}
Esempio n. 3
0
func checkHashes(header http.Header, file string) (bool, error) {
	googHashes := header[http.CanonicalHeaderKey("x-goog-hash")]

	for _, googHash := range googHashes {
		tokens := strings.SplitN(googHash, "=", 2)
		hashType := tokens[0]
		hashValue, err := base64.StdEncoding.DecodeString(tokens[1])
		if err != nil {
			comm.Logf("Could not verify %s hash: %s", hashType, err)
			continue
		}

		start := time.Now()
		checked, err := checkHash(hashType, hashValue, file)
		if err != nil {
			return false, errors.Wrap(err, 1)
		}

		if checked {
			comm.Debugf("%10s pass (took %s)", hashType, time.Since(start))
		} else {
			comm.Debugf("%10s skip (use --thorough to force check)", hashType)
		}
	}

	return true, nil
}
Esempio n. 4
0
func setupHTTPDebug() {
	debugPort := os.Getenv("BUTLER_DEBUG_PORT")

	if debugPort == "" {
		return
	}

	addr := fmt.Sprintf("localhost:%s", debugPort)
	go func() {
		err := http.ListenAndServe(addr, nil)
		if err != nil {
			comm.Logf("http debug error: %s", err.Error())
		}
	}()
	comm.Logf("serving pprof debug interface on %s", addr)
}
Esempio n. 5
0
func doLogout() error {
	var identity = *appArgs.identity

	_, err := os.Lstat(identity)
	if err != nil {
		if os.IsNotExist(err) {
			fmt.Println("No saved credentials at", identity)
			fmt.Println("Nothing to do.")
			return nil
		}
	}

	comm.Notice("Important note", []string{
		"Note: this command will not invalidate the API key itself.",
		"If you wish to revoke it (for example, because it's been compromised), you should do so in your user settings:",
		"",
		fmt.Sprintf("  %s/user/settings\n\n", *appArgs.address),
	})

	comm.Logf("")

	if !comm.YesNo("Do you want to erase your saved API key?") {
		fmt.Println("Okay, not erasing credentials. Bye!")
		return nil
	}

	err = os.Remove(identity)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	fmt.Println("You've successfully erased the API key that was saved on your computer.")

	return nil
}
Esempio n. 6
0
func untar(file string, dir string) {
	settings := archiver.ExtractSettings{
		Consumer: comm.NewStateConsumer(),
	}

	comm.StartProgress()
	res, err := archiver.ExtractTar(file, dir, settings)
	comm.EndProgress()

	must(err)
	comm.Logf("Extracted %d dirs, %d files, %d symlinks", res.Dirs, res.Files, res.Symlinks)
}
Esempio n. 7
0
func doUpgrade(head bool) error {
	if head {
		if !comm.YesNo("Do you want to upgrade to the bleeding-edge version? Things may break!") {
			comm.Logf("Okay, not upgrading. Bye!")
			return nil
		}

		return applyUpgrade("head", "head")
	}

	if version == "head" {
		comm.Statf("Bleeding-edge, not upgrading unless told to.")
		comm.Logf("(Use `--head` if you want to upgrade to the latest bleeding-edge version)")
		return nil
	}

	comm.Opf("Looking for upgrades...")

	currentVer, latestVer, err := queryLatestVersion()
	if err != nil {
		return fmt.Errorf("Version check failed: %s", err.Error())
	}

	if latestVer == nil || currentVer.GTE(*latestVer) {
		comm.Statf("Your butler is up-to-date. Have a nice day!")
		return nil
	}

	comm.Statf("Current version: %s", currentVer.String())
	comm.Statf("Latest version : %s", latestVer.String())

	if !comm.YesNo("Do you want to upgrade now?") {
		comm.Logf("Okay, not upgrading. Bye!")
		return nil
	}

	must(applyUpgrade(currentVer.String(), latestVer.String()))
	return nil
}
Esempio n. 8
0
func doStatus(specStr string) error {
	spec, err := itchio.ParseSpec(specStr)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	client, err := authenticateViaOauth()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	listChannelsResp, err := client.ListChannels(spec.Target)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Channel", "Upload", "Build", "State"})

	found := false

	for _, ch := range listChannelsResp.Channels {
		if spec.Channel != "" && ch.Name != spec.Channel {
			continue
		}
		found = true

		if ch.Head != nil {
			files := ch.Head.Files
			line := []string{ch.Name, fmt.Sprintf("#%d", ch.Upload.ID), buildState(ch.Head), filesState(files)}
			table.Append(line)
		} else {
			line := []string{ch.Name, fmt.Sprintf("#%d", ch.Upload.ID), "No builds yet"}
			table.Append(line)
		}

		if ch.Pending != nil {
			files := ch.Pending.Files
			line := []string{"", "", buildState(ch.Pending), filesState(files)}
			table.Append(line)
		}
	}

	if found {
		table.Render()
	} else {
		comm.Logf("No channel %s found for %s", spec.Channel, spec.Target)
	}

	return nil
}
Esempio n. 9
0
File: fs.go Progetto: itchio/butler
func sizeof(path string) {
	totalSize := int64(0)

	inc := func(_ string, f os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		totalSize += f.Size()
		return nil
	}

	filepath.Walk(path, inc)
	comm.Logf("Total size of %s: %s", path, humanize.IBytes(uint64(totalSize)))
	comm.Result(totalSize)
}
Esempio n. 10
0
File: fs.go Progetto: itchio/butler
func wipe(path string) {
	// why have retry logic built into wipe? sometimes when uninstalling
	// games on windows, the os will randomly return I/O errors, retrying
	// usually helps.
	attempt := 0
	sleepPatterns := []time.Duration{
		time.Millisecond * 200,
		time.Millisecond * 800,
		time.Millisecond * 1600,
		time.Millisecond * 3200,
	}

	for attempt <= len(sleepPatterns) {
		err := tryWipe(path)
		if err == nil {
			break
		}

		if attempt == len(sleepPatterns) {
			comm.Dief("Could not wipe %s: %s", path, err.Error())
		}

		comm.Logf("While wiping %s: %s", path, err.Error())

		comm.Logf("Trying to brute-force permissions, who knows...")
		err = tryChmod(path)
		if err != nil {
			comm.Logf("While bruteforcing: %s", err.Error())
		}

		comm.Logf("Sleeping for a bit before we retry...")
		sleepDuration := sleepPatterns[attempt]
		time.Sleep(sleepDuration)
		attempt++
	}
}
Esempio n. 11
0
func unzip(file string, dir string, resumeFile string) {
	comm.Opf("Extracting zip %s to %s", file, dir)

	settings := archiver.ExtractSettings{
		Consumer:   comm.NewStateConsumer(),
		ResumeFrom: resumeFile,
		OnUncompressedSizeKnown: func(uncompressedSize int64) {
			comm.StartProgressWithTotalBytes(uncompressedSize)
		},
	}

	res, err := archiver.ExtractPath(file, dir, settings)
	comm.EndProgress()

	must(err)
	comm.Logf("Extracted %d dirs, %d files, %d symlinks", res.Dirs, res.Files, res.Symlinks)
}
Esempio n. 12
0
func versionCheck() {
	currentVer, latestVer, err := queryLatestVersion()
	if err != nil {
		comm.Logf("Version check failed: %s", err.Error())
	}

	if currentVer == nil || latestVer == nil {
		return
	}

	if latestVer.GT(*currentVer) {
		comm.Notice("New version available",
			[]string{
				fmt.Sprintf("Current version: %s", version),
				fmt.Sprintf("Latest version:  %s", latestVer),
				"",
				"Run `butler upgrade` to get it.",
			})
	}
}
Esempio n. 13
0
func doProbe(patch string) error {
	patchReader, err := eos.Open(patch)
	if err != nil {
		return err
	}

	defer patchReader.Close()

	stats, err := patchReader.Stat()
	if err != nil {
		return err
	}

	comm.Statf("patch:  %s", humanize.IBytes(uint64(stats.Size())))

	rctx := wire.NewReadContext(patchReader)
	err = rctx.ExpectMagic(pwr.PatchMagic)
	if err != nil {
		return err
	}

	header := &pwr.PatchHeader{}
	err = rctx.ReadMessage(header)
	if err != nil {
		return err
	}

	rctx, err = pwr.DecompressWire(rctx, header.Compression)
	if err != nil {
		return err
	}

	target := &tlc.Container{}
	err = rctx.ReadMessage(target)
	if err != nil {
		return err
	}

	source := &tlc.Container{}
	err = rctx.ReadMessage(source)
	if err != nil {
		return err
	}

	comm.Statf("target: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats())
	comm.Statf("source: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats())

	var patchStats []patchStat

	sh := &pwr.SyncHeader{}
	rop := &pwr.SyncOp{}

	for fileIndex, f := range source.Files {
		stat := patchStat{
			fileIndex: int64(fileIndex),
			freshData: f.Size,
		}

		sh.Reset()
		err = rctx.ReadMessage(sh)
		if err != nil {
			return err
		}

		if sh.FileIndex != int64(fileIndex) {
			return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex)
		}

		readingOps := true

		var pos int64

		for readingOps {
			rop.Reset()

			err = rctx.ReadMessage(rop)
			if err != nil {
				return err
			}

			switch rop.Type {
			case pwr.SyncOp_BLOCK_RANGE:
				fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize
				lastIndex := rop.BlockIndex + (rop.BlockSpan - 1)
				lastSize := pwr.ComputeBlockSize(f.Size, lastIndex)
				totalSize := (fixedSize + lastSize)
				stat.freshData -= totalSize
				pos += totalSize
			case pwr.SyncOp_DATA:
				totalSize := int64(len(rop.Data))
				if *appArgs.verbose {
					comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)),
						pos, pos+totalSize)
				}
				pos += totalSize
			case pwr.SyncOp_HEY_YOU_DID_IT:
				readingOps = false
			}
		}

		patchStats = append(patchStats, stat)
	}

	sort.Sort(byDecreasingFreshData(patchStats))

	var totalFresh int64
	for _, stat := range patchStats {
		totalFresh += stat.freshData
	}

	var eightyFresh = int64(0.8 * float64(totalFresh))
	var printedFresh int64

	comm.Opf("80%% of fresh data is in the following files:")

	for _, stat := range patchStats {
		f := source.Files[stat.fileIndex]
		comm.Logf("%s in %s (%.2f%% changed)",
			humanize.IBytes(uint64(stat.freshData)),
			f.Path,
			float64(stat.freshData)/float64(f.Size)*100.0)

		printedFresh += stat.freshData
		if printedFresh >= eightyFresh {
			break
		}
	}

	return nil
}
Esempio n. 14
0
File: cp.go Progetto: itchio/butler
func doCp(srcPath string, destPath string, resume bool) error {
	src, err := eos.Open(srcPath)
	if err != nil {
		return err
	}

	defer src.Close()

	dir := filepath.Dir(destPath)
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return err
	}

	flags := os.O_CREATE | os.O_WRONLY

	dest, err := os.OpenFile(destPath, flags, 0644)
	if err != nil {
		return err
	}

	defer dest.Close()

	stats, err := src.Stat()
	if err != nil {
		return err
	}

	totalBytes := int64(stats.Size())
	startOffset := int64(0)

	if resume {
		startOffset, err = dest.Seek(0, os.SEEK_END)
		if err != nil {
			return err
		}

		if startOffset == 0 {
			comm.Logf("Downloading %s", humanize.IBytes(uint64(totalBytes)))
		} else if startOffset > totalBytes {
			comm.Logf("Existing data too big (%s > %s), starting over", humanize.IBytes(uint64(startOffset)), humanize.IBytes(uint64(totalBytes)))
		} else if startOffset == totalBytes {
			comm.Logf("All %s already there", humanize.IBytes(uint64(totalBytes)))
			return nil
		}

		comm.Logf("Resuming at %s / %s", humanize.IBytes(uint64(startOffset)), humanize.IBytes(uint64(totalBytes)))

		_, err = src.Seek(startOffset, os.SEEK_SET)
		if err != nil {
			return err
		}
	} else {
		comm.Logf("Downloading %s", humanize.IBytes(uint64(totalBytes)))
	}

	start := time.Now()

	comm.Progress(float64(startOffset) / float64(totalBytes))
	comm.StartProgressWithTotalBytes(totalBytes)

	cw := counter.NewWriterCallback(func(count int64) {
		alpha := float64(startOffset+count) / float64(totalBytes)
		comm.Progress(alpha)
	}, dest)

	copiedBytes, err := io.Copy(cw, src)
	if err != nil {
		return err
	}
	comm.EndProgress()

	totalDuration := time.Since(start)
	prettyStartOffset := humanize.IBytes(uint64(startOffset))
	prettySize := humanize.IBytes(uint64(copiedBytes))
	perSecond := humanize.IBytes(uint64(float64(totalBytes-startOffset) / totalDuration.Seconds()))
	comm.Statf("%s + %s copied @ %s/s\n", prettyStartOffset, prettySize, perSecond)

	return nil
}
Esempio n. 15
0
File: dl.go Progetto: itchio/butler
func tryDl(url string, dest string) (int64, error) {
	existingBytes := int64(0)
	stats, err := os.Lstat(dest)
	if err == nil {
		existingBytes = stats.Size()
	}

	client := timeout.NewDefaultClient()

	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", userAgent())
	byteRange := fmt.Sprintf("bytes=%d-", existingBytes)

	req.Header.Set("Range", byteRange)
	resp, err := client.Do(req)
	if err != nil {
		return 0, errors.Wrap(err, 1)
	}
	defer resp.Body.Close()

	doDownload := true
	totalBytes := existingBytes + resp.ContentLength

	hostInfo := fmt.Sprintf("%s at %s", resp.Header.Get("Server"), req.Host)

	switch resp.StatusCode {
	case 200: // OK
		comm.Debugf("HTTP 200 OK (no byte range support)")
		totalBytes = resp.ContentLength

		if existingBytes == resp.ContentLength {
			// already have the exact same number of bytes, hopefully the same ones
			doDownload = false
		} else {
			// will send data, but doesn't support byte ranges
			existingBytes = 0
			os.Truncate(dest, 0)
		}
	case 206: // Partial Content
		comm.Debugf("HTTP 206 Partial Content")
		// will send incremental data
	case 416: // Requested Range not Satisfiable
		comm.Debugf("HTTP 416 Requested Range not Satisfiable")
		// already has everything
		doDownload = false

		req, _ := http.NewRequest("HEAD", url, nil)
		req.Header.Set("User-Agent", userAgent())
		resp, err = client.Do(req)
		if err != nil {
			return 0, errors.Wrap(err, 1)
		}

		if existingBytes > resp.ContentLength {
			comm.Debugf("Existing file too big (%d), truncating to %d", existingBytes, resp.ContentLength)
			existingBytes = resp.ContentLength
			os.Truncate(dest, existingBytes)
		}
		totalBytes = existingBytes
	default:
		return 0, fmt.Errorf("%s responded with HTTP %s", hostInfo, resp.Status)
	}

	if doDownload {
		if existingBytes > 0 {
			comm.Logf("Resuming (%s + %s = %s) download from %s", humanize.IBytes(uint64(existingBytes)), humanize.IBytes(uint64(resp.ContentLength)), humanize.IBytes(uint64(totalBytes)), hostInfo)
		} else {
			comm.Logf("Downloading %s from %s", humanize.IBytes(uint64(resp.ContentLength)), hostInfo)
		}
		err = appendAllToFile(resp.Body, dest, existingBytes, totalBytes)
		if err != nil {
			return 0, errors.Wrap(err, 1)
		}
	} else {
		comm.Log("Already fully downloaded")
	}

	_, err = checkIntegrity(resp, totalBytes, dest)
	if err != nil {
		comm.Log("Integrity checks failed, truncating")
		os.Truncate(dest, 0)
		return 0, errors.Wrap(err, 1)
	}

	return totalBytes, nil
}
Esempio n. 16
0
func file(path string) {
	reader, err := eos.Open(path)
	must(err)

	defer reader.Close()

	stats, err := reader.Stat()
	if os.IsNotExist(err) {
		comm.Dief("%s: no such file or directory", path)
	}
	must(err)

	if stats.IsDir() {
		comm.Logf("%s: directory", path)
		return
	}

	if stats.Size() == 0 {
		comm.Logf("%s: empty file. peaceful.", path)
		return
	}

	prettySize := humanize.IBytes(uint64(stats.Size()))

	var magic int32
	must(binary.Read(reader, wire.Endianness, &magic))

	switch magic {
	case pwr.PatchMagic:
		{
			ph := &pwr.PatchHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(ph))

			rctx, err = pwr.DecompressWire(rctx, ph.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container)) // target container
			container.Reset()
			must(rctx.ReadMessage(container)) // source container

			comm.Logf("%s: %s wharf patch file (%s) with %s", path, prettySize, ph.GetCompression().ToString(), container.Stats())
			comm.Result(ContainerResult{
				Type:             "wharf/patch",
				NumFiles:         len(container.Files),
				NumDirs:          len(container.Dirs),
				NumSymlinks:      len(container.Symlinks),
				UncompressedSize: container.Size,
			})
		}

	case pwr.SignatureMagic:
		{
			sh := &pwr.SignatureHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(sh))

			rctx, err = pwr.DecompressWire(rctx, sh.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))

			comm.Logf("%s: %s wharf signature file (%s) with %s", path, prettySize, sh.GetCompression().ToString(), container.Stats())
			comm.Result(ContainerResult{
				Type:             "wharf/signature",
				NumFiles:         len(container.Files),
				NumDirs:          len(container.Dirs),
				NumSymlinks:      len(container.Symlinks),
				UncompressedSize: container.Size,
			})
		}

	case pwr.ManifestMagic:
		{
			mh := &pwr.ManifestHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(mh))

			rctx, err = pwr.DecompressWire(rctx, mh.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))

			comm.Logf("%s: %s wharf manifest file (%s) with %s", path, prettySize, mh.GetCompression().ToString(), container.Stats())
			comm.Result(ContainerResult{
				Type:             "wharf/manifest",
				NumFiles:         len(container.Files),
				NumDirs:          len(container.Dirs),
				NumSymlinks:      len(container.Symlinks),
				UncompressedSize: container.Size,
			})
		}

	case pwr.WoundsMagic:
		{
			wh := &pwr.WoundsHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(wh))

			container := &tlc.Container{}
			must(rctx.ReadMessage(container))

			files := make(map[int64]bool)
			totalWounds := int64(0)

			for {
				wound := &pwr.Wound{}

				err = rctx.ReadMessage(wound)
				if err != nil {
					if errors.Is(err, io.EOF) {
						break
					} else {
						must(err)
					}
				}

				if wound.Kind == pwr.WoundKind_FILE {
					totalWounds += (wound.End - wound.Start)
					files[wound.Index] = true
				}
			}

			comm.Logf("%s: %s wharf wounds file with %s, %s wounds in %d files", path, prettySize, container.Stats(),
				humanize.IBytes(uint64(totalWounds)), len(files))
			comm.Result(ContainerResult{
				Type: "wharf/wounds",
			})
		}

	default:
		_, err := reader.Seek(0, os.SEEK_SET)
		must(err)

		wasZip := func() bool {
			zr, err := zip.NewReader(reader, stats.Size())
			if err != nil {
				if err != zip.ErrFormat {
					must(err)
				}
				return false
			}

			container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
			must(err)

			comm.Logf("%s: %s zip file with %s", path, prettySize, container.Stats())
			comm.Result(ContainerResult{
				Type:             "zip",
				NumFiles:         len(container.Files),
				NumDirs:          len(container.Dirs),
				NumSymlinks:      len(container.Symlinks),
				UncompressedSize: container.Size,
			})
			return true
		}()

		if !wasZip {
			comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
		}
	}
}
Esempio n. 17
0
func doApply(patch string, target string, output string, inplace bool, signaturePath string, woundsPath string) error {
	if output == "" {
		output = target
	}

	target = path.Clean(target)
	output = path.Clean(output)
	if output == target {
		if !inplace {
			comm.Dief("Refusing to destructively patch %s without --inplace", output)
		}
	}

	if signaturePath == "" {
		comm.Opf("Patching %s", output)
	} else {
		comm.Opf("Patching %s with validation", output)
	}

	startTime := time.Now()

	patchReader, err := eos.Open(patch)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	var signature *pwr.SignatureInfo
	if signaturePath != "" {
		sigReader, sigErr := eos.Open(signaturePath)
		if sigErr != nil {
			return errors.Wrap(sigErr, 1)
		}
		defer sigReader.Close()

		signature, sigErr = pwr.ReadSignature(sigReader)
		if sigErr != nil {
			return errors.Wrap(sigErr, 1)
		}
	}

	actx := &pwr.ApplyContext{
		TargetPath: target,
		OutputPath: output,
		InPlace:    inplace,
		Signature:  signature,
		WoundsPath: woundsPath,

		Consumer: comm.NewStateConsumer(),
	}

	comm.StartProgress()
	err = actx.ApplyPatch(patchReader)
	if err != nil {
		return errors.Wrap(err, 1)
	}
	comm.EndProgress()

	container := actx.SourceContainer
	prettySize := humanize.IBytes(uint64(container.Size))
	perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))

	if actx.InPlace {
		statStr := ""
		if actx.Stats.TouchedFiles > 0 {
			statStr += fmt.Sprintf("patched %d, ", actx.Stats.TouchedFiles)
		}
		if actx.Stats.MovedFiles > 0 {
			statStr += fmt.Sprintf("renamed %d, ", actx.Stats.MovedFiles)
		}
		if actx.Stats.DeletedFiles > 0 {
			statStr += fmt.Sprintf("deleted %d, ", actx.Stats.DeletedFiles)
		}

		comm.Statf("%s (%s stage)", statStr, humanize.IBytes(uint64(actx.Stats.StageSize)))
	}
	comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond)

	if actx.WoundsConsumer != nil && actx.WoundsConsumer.HasWounds() {
		extra := ""
		if actx.WoundsPath != "" {
			extra = fmt.Sprintf(" (written to %s)", actx.WoundsPath)
		}

		totalCorrupted := actx.WoundsConsumer.TotalCorrupted()
		comm.Logf("Result has wounds, %s corrupted data%s", humanize.IBytes(uint64(totalCorrupted)), extra)
	}

	return nil
}
Esempio n. 18
0
File: fs.go Progetto: itchio/butler
// Does not preserve users, nor permission, except the executable bit
func ditto(src string, dst string) {
	comm.Debugf("rsync -a %s %s", src, dst)

	totalSize := int64(0)
	doneSize := int64(0)
	oldProgress := 0.0

	inc := func(_ string, f os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		totalSize += f.Size()
		return nil
	}

	onFile := func(path string, f os.FileInfo, err error) error {
		if err != nil {
			comm.Logf("ignoring error %s", err.Error())
			return nil
		}

		rel, err := filepath.Rel(src, path)
		must(err)

		dstpath := filepath.Join(dst, rel)
		mode := f.Mode()

		switch {
		case mode.IsDir():
			dittoMkdir(dstpath)

		case mode.IsRegular():
			dittoReg(path, dstpath, os.FileMode(f.Mode()&archiver.LuckyMode|archiver.ModeMask))

		case (mode&os.ModeSymlink > 0):
			dittoSymlink(path, dstpath, f)
		}

		comm.Debug(rel)

		doneSize += f.Size()

		progress := float64(doneSize) / float64(totalSize)
		if progress-oldProgress > 0.01 {
			oldProgress = progress
			comm.Progress(progress)
		}

		return nil
	}

	rootinfo, err := os.Lstat(src)
	must(err)

	if rootinfo.IsDir() {
		totalSize = 0
		comm.Logf("Counting files in %s...", src)
		filepath.Walk(src, inc)

		comm.Logf("Mirroring...")
		filepath.Walk(src, onFile)
	} else {
		totalSize = rootinfo.Size()
		onFile(src, rootinfo, nil)
	}

	comm.EndProgress()
}
Esempio n. 19
0
func doPush(buildPath string, specStr string, userVersion string, fixPerms bool) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan walkResult)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs, fixPerms)

	spec, err := itchio.ParseSpec(specStr)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = spec.EnsureChannel()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	client, err := authenticateViaOauth()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	newBuildRes, err := client.CreateBuild(spec.Target, spec.Channel, userVersion)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature *pwr.SignatureInfo

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", spec.Channel)
		targetSignature = &pwr.SignatureInfo{
			Container: &tlc.Container{},
			Hashes:    make([]wsync.BlockHash, 0),
		}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", spec.Channel, parentID)
		var buildFiles itchio.ListBuildFilesResponse
		buildFiles, err = client.ListBuildFiles(parentID)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		signatureFile := itchio.FindBuildFile(itchio.BuildFileType_SIGNATURE, buildFiles.Files)
		if signatureFile == nil {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		var signatureReader io.Reader
		signatureReader, err = client.DownloadBuildFile(parentID, signatureFile.ID)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return errors.Wrap(err, 1)
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	uploadDone := make(chan bool)
	uploadErrs := make(chan error)

	patchWriter, err := uploader.NewResumableUpload(newPatchRes.File.UploadURL,
		uploadDone, uploadErrs, uploader.ResumableUploadSettings{
			Consumer: comm.NewStateConsumer(),
		})
	patchWriter.MaxChunkGroup = *appArgs.maxChunkGroup
	if err != nil {
		return errors.Wrap(err, 1)
	}

	signatureWriter, err := uploader.NewResumableUpload(newSignatureRes.File.UploadURL,
		uploadDone, uploadErrs, uploader.ResumableUploadSettings{
			Consumer: comm.NewStateConsumer(),
		})
	signatureWriter.MaxChunkGroup = *appArgs.maxChunkGroup
	if err != nil {
		return errors.Wrap(err, 1)
	}

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.
	// note that we could actually start diffing before all the file
	// creation & upload setup is done

	var sourceContainer *tlc.Container
	var sourcePool wsync.Pool

	comm.Debugf("Waiting for source container")
	select {
	case walkErr := <-walkErrs:
		return errors.Wrap(walkErr, 1)
	case walkies := <-sourceContainerChan:
		comm.Debugf("Got sourceContainer!")
		sourceContainer = walkies.container
		sourcePool = walkies.pool
		break
	}

	comm.Opf("Pushing %s (%s)", humanize.IBytes(uint64(sourceContainer.Size)), sourceContainer.Stats())

	comm.Debugf("Building diff context")
	var readBytes int64

	bytesPerSec := float64(0)
	lastUploadedBytes := int64(0)
	stopTicking := make(chan struct{})

	updateProgress := func() {
		uploadedBytes := int64(float64(patchWriter.UploadedBytes))

		// input bytes that aren't in output, for example:
		//  - bytes that have been compressed away
		//  - bytes that were in old build and were simply reused
		goneBytes := readBytes - patchWriter.TotalBytes

		conservativeTotalBytes := sourceContainer.Size - goneBytes

		leftBytes := conservativeTotalBytes - uploadedBytes
		if leftBytes > AlmostThereThreshold {
			netStatus := "- network idle"
			if bytesPerSec > 1 {
				netStatus = fmt.Sprintf("@ %s/s", humanize.IBytes(uint64(bytesPerSec)))
			}
			comm.ProgressLabel(fmt.Sprintf("%s, %s left", netStatus, humanize.IBytes(uint64(leftBytes))))
		} else {
			comm.ProgressLabel(fmt.Sprintf("- almost there"))
		}

		conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
		conservativeProgress = min(1.0, conservativeProgress)
		comm.Progress(conservativeProgress)

		comm.ProgressScale(float64(readBytes) / float64(sourceContainer.Size))
	}

	go func() {
		ticker := time.NewTicker(time.Second * time.Duration(2))
		for {
			select {
			case <-ticker.C:
				bytesPerSec = float64(patchWriter.UploadedBytes-lastUploadedBytes) / 2.0
				lastUploadedBytes = patchWriter.UploadedBytes
				updateProgress()
			case <-stopTicking:
				break
			}
		}
	}()

	patchWriter.OnProgress = updateProgress

	stateConsumer := &state.Consumer{
		OnProgress: func(progress float64) {
			readBytes = int64(float64(sourceContainer.Size) * progress)
			updateProgress()
		},
	}

	dctx := &pwr.DiffContext{
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   1,
		},

		SourceContainer: sourceContainer,
		Pool:            sourcePool,

		TargetContainer: targetSignature.Container,
		TargetSignature: targetSignature.Hashes,

		Consumer: stateConsumer,
	}

	comm.StartProgress()
	comm.ProgressScale(0.0)
	err = dctx.WritePatch(patchCounter, signatureCounter)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	// close in a goroutine to avoid deadlocking
	doClose := func(c io.Closer, done chan bool, errs chan error) {
		closeErr := c.Close()
		if closeErr != nil {
			errs <- errors.Wrap(closeErr, 1)
			return
		}

		done <- true
	}

	go doClose(patchWriter, uploadDone, uploadErrs)
	go doClose(signatureWriter, uploadDone, uploadErrs)

	for c := 0; c < 4; c++ {
		select {
		case uploadErr := <-uploadErrs:
			return errors.Wrap(uploadErr, 1)
		case <-uploadDone:
			comm.Debugf("upload done")
		}
	}

	close(stopTicking)
	comm.ProgressLabel("finalizing build")

	finalDone := make(chan bool)
	finalErrs := make(chan error)

	doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
		_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		done <- true
	}

	go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
	go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)

	for i := 0; i < 2; i++ {
		select {
		case err := <-finalErrs:
			return errors.Wrap(err, 1)
		case <-finalDone:
		}
	}

	comm.EndProgress()

	{
		prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes))
		savings := 100.0 - relToNew

		if dctx.ReusedBytes > 0 {
			comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		} else {
			comm.Statf("Added %s fresh data", prettyFreshSize)
		}

		if savings > 0 && !math.IsNaN(savings) {
			comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
		} else {
			comm.Statf("%s patch (no savings)", prettyPatchSize)
		}
	}
	comm.Opf("Build is now processing, should be up in a bit (see `butler status`)")
	comm.Logf("")

	return nil
}
Esempio n. 20
0
func ls(path string) {
	reader, err := eos.Open(path)
	must(err)

	defer reader.Close()

	stats, err := reader.Stat()
	if os.IsNotExist(err) {
		comm.Dief("%s: no such file or directory", path)
	}
	must(err)

	if stats.IsDir() {
		comm.Logf("%s: directory", path)
		return
	}

	if stats.Size() == 0 {
		comm.Logf("%s: empty file. peaceful.", path)
		return
	}

	log := func(line string) {
		comm.Logf(line)
	}

	var magic int32
	must(binary.Read(reader, wire.Endianness, &magic))

	switch magic {
	case pwr.PatchMagic:
		{
			h := &pwr.PatchHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			log("pre-patch container:")
			container.Print(log)

			container.Reset()
			must(rctx.ReadMessage(container))
			log("================================")
			log("post-patch container:")
			container.Print(log)
		}

	case pwr.SignatureMagic:
		{
			h := &pwr.SignatureHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			container.Print(log)
		}

	case pwr.ManifestMagic:
		{
			h := &pwr.ManifestHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			container.Print(log)
		}

	case pwr.WoundsMagic:
		{
			wh := &pwr.WoundsHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(wh))

			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			container.Print(log)

			for {
				wound := &pwr.Wound{}
				err = rctx.ReadMessage(wound)
				if err != nil {
					if errors.Is(err, io.EOF) {
						break
					} else {
						must(err)
					}
				}
				comm.Logf(wound.PrettyString(container))
			}
		}

	default:
		_, err := reader.Seek(0, os.SEEK_SET)
		must(err)

		wasZip := func() bool {
			zr, err := zip.NewReader(reader, stats.Size())
			if err != nil {
				if err != zip.ErrFormat {
					must(err)
				}
				return false
			}

			container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
			must(err)
			container.Print(log)
			return true
		}()

		if !wasZip {
			comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
		}
	}
}