func main() { app := cli.NewApp() app.Email = "*****@*****.**" app.Name = "ipfs-bench" app.Action = func(context *cli.Context) error { verbose = context.Bool("verbose") noclean = context.Bool("noclean") params := &MultinodeParams{ NumNodes: context.Int("numnodes"), FileSize: context.Int("filesize"), Net: &cn.LinkSettings{ Latency: context.Int("latency"), }, } res, err := RunMultinode(params) if err != nil { perr("error running tests: %s", err) return nil } for _, f := range res.FetchStats { hb := hum.IBytes(uint64(f.BW)) fmt.Printf("Took: %s, Average BW: %s, TX Size: %d, DupBlocks: %d\n", f.Duration, hb, f.Total, f.DupBlocks) } fmt.Println(hum.IBytes(uint64(res.AverageBandwidth()))) return nil } app.Flags = []cli.Flag{ cli.IntFlag{ Name: "filesize", Usage: "specify size of file to transfer", Value: 1000000, }, cli.IntFlag{ Name: "numnodes", Usage: "number of nodes to run test with", Value: 5, }, cli.IntFlag{ Name: "latency", Usage: "set per-link latency", Value: 0, }, cli.BoolFlag{ Name: "verbose", Usage: "print verbose logging info", }, cli.BoolFlag{ Name: "noclean", Usage: "do not clean up docker nodes after test", }, } app.Run(os.Args) }
// Get formatted disk/storage info message. func getStorageInfoMsg(storageInfo StorageInfo) string { msg := fmt.Sprintf("%s %s Free, %s Total", colorBlue("Drive Capacity:"), humanize.IBytes(uint64(storageInfo.Free)), humanize.IBytes(uint64(storageInfo.Total))) if storageInfo.Backend.Type == XL { diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks, storageInfo.Backend.OfflineDisks) if maxDiskFailures := storageInfo.Backend.ReadQuorum - storageInfo.Backend.OfflineDisks; maxDiskFailures >= 0 { diskInfo += fmt.Sprintf("We can withstand [%d] more drive failure(s).", maxDiskFailures) } msg += colorBlue("\nStatus:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo) } return msg }
func collectMem() (*Data, error) { v, err := mem.VirtualMemory() return &Data{ Name: "mem", Data: map[string]interface{}{ "total": v.Total, "free": v.Free, "used": v.Used, "summary": fmt.Sprintf("Total: %v, Free %v, Used %v", humanize.IBytes(v.Total), humanize.IBytes(v.Free), humanize.IBytes(v.Used)), }, }, err }
// Constructs a formatted heal message, when cluster is found to be in state where it requires healing. // healing is optional, server continues to initialize object layer after printing this message. // it is upto the end user to perform a heal if needed. func getHealMsg(endpoints []*url.URL, storageDisks []StorageAPI) string { msg := fmt.Sprintln("\nData volume requires HEALING. Healing is not implemented yet stay tuned:") // FIXME: Enable this after we bring in healing. // msg += "MINIO_ACCESS_KEY=%s " // msg += "MINIO_SECRET_KEY=%s " // msg += "minio control heal %s" // creds := serverConfig.GetCredential() // msg = fmt.Sprintf(msg, creds.AccessKeyID, creds.SecretAccessKey, getHealEndpoint(isSSL(), endpoints[0])) disksInfo, _, _ := getDisksInfo(storageDisks) for i, info := range disksInfo { if storageDisks[i] == nil { continue } msg += fmt.Sprintf( "\n[%s] %s - %s %s", formatInts(i+1, len(storageDisks)), endpoints[i], humanize.IBytes(uint64(info.Total)), func() string { if info.Total > 0 { return "online" } return "offline" }(), ) } return msg }
func benchAddSize(n *core.IpfsNode, cfg *BenchCfg, size int64) error { f := func(b *testing.B) { b.SetBytes(size) for i := 0; i < b.N; i++ { r := io.LimitReader(randbo.New(), size) spl := chunk.NewSizeSplitter(r, cfg.Blocksize) _, err := importer.BuildDagFromReader(n.DAG, spl, nil) if err != nil { fmt.Printf("ERRROR: ", err) b.Fatal(err) } } } br := testing.Benchmark(f) bs := humanize.IBytes(uint64(size)) fmt.Printf("Add File (%s):\t%s\n", bs, br) err := cr.GarbageCollect(n, context.Background()) if err != nil { return err } return nil }
// PrettyString returns a human-readable English string for a given wound func (w *Wound) PrettyString(container *tlc.Container) string { switch w.Kind { case WoundKind_DIR: dir := container.Dirs[w.Index] return fmt.Sprintf("directory wound (%s should exist)", dir.Path) case WoundKind_SYMLINK: symlink := container.Symlinks[w.Index] return fmt.Sprintf("symlink wound (%s should point to %s)", symlink.Path, symlink.Dest) case WoundKind_FILE: file := container.Files[w.Index] woundSize := humanize.IBytes(uint64(w.End - w.Start)) offset := humanize.IBytes(uint64(w.Start)) return fmt.Sprintf("~%s wound %s into %s", woundSize, offset, file.Path) default: return fmt.Sprintf("unknown wound (%d)", w.Kind) } }
func (u watchMessage) String() string { msg := console.Colorize("Time", fmt.Sprintf("[%s] ", u.Event.Time)) if u.Event.Type == EventCreate { msg += console.Colorize("Size", fmt.Sprintf("%6s ", humanize.IBytes(uint64(u.Event.Size)))) } else { msg += fmt.Sprintf("%6s ", "") } msg += console.Colorize("EventType", fmt.Sprintf("%s ", u.Event.Type)) msg += console.Colorize("ObjectName", fmt.Sprintf("%s", u.Event.Path)) return msg }
func sizeof(path string) { totalSize := int64(0) inc := func(_ string, f os.FileInfo, err error) error { if err != nil { return nil } totalSize += f.Size() return nil } filepath.Walk(path, inc) comm.Logf("Total size of %s: %s", path, humanize.IBytes(uint64(totalSize))) comm.Result(totalSize) }
func benchDiskWriteSize(dir string, size int64) error { benchdir := path.Join(dir, fmt.Sprintf("benchfiles-%d", size)) err := os.Mkdir(benchdir, 0777) if err != nil { return err } n := 0 f := func(b *testing.B) { b.SetBytes(size) r := randbo.New() for i := 0; i < b.N; i++ { n++ fi, err := os.Create(path.Join(dir, fmt.Sprint(n))) if err != nil { fmt.Println(err) b.Fatal(err) } _, err = io.CopyN(fi, r, size) if err != nil { fi.Close() fmt.Println(err) b.Fatal(err) } fi.Close() } } br := testing.Benchmark(f) bs := humanize.IBytes(uint64(size)) fmt.Printf("DiskWrite (%s):\t%s\n", bs, br) err = os.RemoveAll(benchdir) if err != nil { return err } return nil }
// Constructs a formatted regular message when we have sufficient disks to start the cluster. func getStorageInitMsg(titleMsg string, endpoints []*url.URL, storageDisks []StorageAPI) string { msg := colorBlue(titleMsg) disksInfo, _, _ := getDisksInfo(storageDisks) for i, info := range disksInfo { if storageDisks[i] == nil { continue } msg += fmt.Sprintf( "\n[%s] %s - %s %s", formatInts(i+1, len(storageDisks)), endpoints[i], humanize.IBytes(uint64(info.Total)), func() string { if info.Total > 0 { return "online" } return "offline" }(), ) } return msg }
// Do computes the difference between old and new, according to the bsdiff // algorithm, and writes the result to patch. func (ctx *DiffContext) Do(old, new io.Reader, writeMessage WriteMessageFunc, consumer *state.Consumer) error { var memstats *runtime.MemStats if ctx.MeasureMem { memstats = &runtime.MemStats{} runtime.ReadMemStats(memstats) consumer.Debugf("Allocated bytes at start of bsdiff: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc))) } if ctx.db == nil { ctx.db = make([]byte, MaxMessageSize) } if ctx.eb == nil { ctx.eb = make([]byte, MaxMessageSize) } obuf, err := ioutil.ReadAll(old) if err != nil { return err } if int64(len(obuf)) > MaxFileSize { return fmt.Errorf("bsdiff: old file too large (%s > %s)", humanize.IBytes(uint64(len(obuf))), humanize.IBytes(uint64(MaxFileSize))) } obuflen := int32(len(obuf)) nbuf, err := ioutil.ReadAll(new) if err != nil { return err } if int64(len(nbuf)) > MaxFileSize { // TODO: provide a different (int64) codepath for >=2GB files return fmt.Errorf("bsdiff: new file too large (%s > %s)", humanize.IBytes(uint64(len(nbuf))), humanize.IBytes(uint64(MaxFileSize))) } nbuflen := int32(len(nbuf)) if ctx.MeasureMem { runtime.ReadMemStats(memstats) consumer.Debugf("Allocated bytes after ReadAll: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc))) } var lenf int32 startTime := time.Now() I := qsufsort(obuf, ctx, consumer) duration := time.Since(startTime) consumer.Debugf("Suffix sorting done in %s", duration) if ctx.MeasureMem { runtime.ReadMemStats(memstats) consumer.Debugf("Allocated bytes after qsufsort: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc))) } // FIXME: the streaming format allows us to allocate less than that db := make([]byte, len(nbuf)) eb := make([]byte, len(nbuf)) bsdc := &Control{} consumer.ProgressLabel("Scanning...") // Compute the differences, writing ctrl as we go var scan, pos, length int32 var lastscan, lastpos, lastoffset int32 for scan < nbuflen { var oldscore int32 scan += length progress := float64(scan) / float64(nbuflen) consumer.Progress(progress) for scsc := scan; scan < nbuflen; scan++ { pos, length = search(I, obuf, nbuf[scan:], 0, obuflen) for ; scsc < scan+length; scsc++ { if scsc+lastoffset < obuflen && obuf[scsc+lastoffset] == nbuf[scsc] { oldscore++ } } if (length == oldscore && length != 0) || length > oldscore+8 { break } if scan+lastoffset < obuflen && obuf[scan+lastoffset] == nbuf[scan] { oldscore-- } } if length != oldscore || scan == nbuflen { var s, Sf int32 lenf = 0 for i := int32(0); lastscan+i < scan && lastpos+i < obuflen; { if obuf[lastpos+i] == nbuf[lastscan+i] { s++ } i++ if s*2-i > Sf*2-lenf { Sf = s lenf = i } } lenb := int32(0) if scan < nbuflen { var s, Sb int32 for i := int32(1); (scan >= lastscan+i) && (pos >= i); i++ { if obuf[pos-i] == nbuf[scan-i] { s++ } if s*2-i > Sb*2-lenb { Sb = s lenb = i } } } if lastscan+lenf > scan-lenb { overlap := (lastscan + lenf) - (scan - lenb) s := int32(0) Ss := int32(0) lens := int32(0) for i := int32(0); i < overlap; i++ { if nbuf[lastscan+lenf-overlap+i] == obuf[lastpos+lenf-overlap+i] { s++ } if nbuf[scan-lenb+i] == obuf[pos-lenb+i] { s-- } if s > Ss { Ss = s lens = i + 1 } } lenf += lens - overlap lenb -= lens } for i := int32(0); i < lenf; i++ { db[i] = nbuf[lastscan+i] - obuf[lastpos+i] } for i := int32(0); i < (scan-lenb)-(lastscan+lenf); i++ { eb[i] = nbuf[lastscan+lenf+i] } bsdc.Add = db[:lenf] bsdc.Copy = eb[:(scan-lenb)-(lastscan+lenf)] bsdc.Seek = int64((pos - lenb) - (lastpos + lenf)) err := writeMessage(bsdc) if err != nil { return err } lastscan = scan - lenb lastpos = pos - lenb lastoffset = pos - scan } } if ctx.MeasureMem { runtime.ReadMemStats(memstats) consumer.Debugf("Allocated bytes after scan: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc))) } bsdc.Reset() bsdc.Eof = true err = writeMessage(bsdc) if err != nil { return err } return nil }
func doProbe(patch string) error { patchReader, err := eos.Open(patch) if err != nil { return err } defer patchReader.Close() stats, err := patchReader.Stat() if err != nil { return err } comm.Statf("patch: %s", humanize.IBytes(uint64(stats.Size()))) rctx := wire.NewReadContext(patchReader) err = rctx.ExpectMagic(pwr.PatchMagic) if err != nil { return err } header := &pwr.PatchHeader{} err = rctx.ReadMessage(header) if err != nil { return err } rctx, err = pwr.DecompressWire(rctx, header.Compression) if err != nil { return err } target := &tlc.Container{} err = rctx.ReadMessage(target) if err != nil { return err } source := &tlc.Container{} err = rctx.ReadMessage(source) if err != nil { return err } comm.Statf("target: %s in %s", humanize.IBytes(uint64(target.Size)), target.Stats()) comm.Statf("source: %s in %s", humanize.IBytes(uint64(target.Size)), source.Stats()) var patchStats []patchStat sh := &pwr.SyncHeader{} rop := &pwr.SyncOp{} for fileIndex, f := range source.Files { stat := patchStat{ fileIndex: int64(fileIndex), freshData: f.Size, } sh.Reset() err = rctx.ReadMessage(sh) if err != nil { return err } if sh.FileIndex != int64(fileIndex) { return fmt.Errorf("malformed patch: expected file %d, got %d", fileIndex, sh.FileIndex) } readingOps := true var pos int64 for readingOps { rop.Reset() err = rctx.ReadMessage(rop) if err != nil { return err } switch rop.Type { case pwr.SyncOp_BLOCK_RANGE: fixedSize := (rop.BlockSpan - 1) * pwr.BlockSize lastIndex := rop.BlockIndex + (rop.BlockSpan - 1) lastSize := pwr.ComputeBlockSize(f.Size, lastIndex) totalSize := (fixedSize + lastSize) stat.freshData -= totalSize pos += totalSize case pwr.SyncOp_DATA: totalSize := int64(len(rop.Data)) if *appArgs.verbose { comm.Debugf("%s fresh data at %s (%d-%d)", humanize.IBytes(uint64(totalSize)), humanize.IBytes(uint64(pos)), pos, pos+totalSize) } pos += totalSize case pwr.SyncOp_HEY_YOU_DID_IT: readingOps = false } } patchStats = append(patchStats, stat) } sort.Sort(byDecreasingFreshData(patchStats)) var totalFresh int64 for _, stat := range patchStats { totalFresh += stat.freshData } var eightyFresh = int64(0.8 * float64(totalFresh)) var printedFresh int64 comm.Opf("80%% of fresh data is in the following files:") for _, stat := range patchStats { f := source.Files[stat.fileIndex] comm.Logf("%s in %s (%.2f%% changed)", humanize.IBytes(uint64(stat.freshData)), f.Path, float64(stat.freshData)/float64(f.Size)*100.0) printedFresh += stat.freshData if printedFresh >= eightyFresh { break } } return nil }
func doCp(srcPath string, destPath string, resume bool) error { src, err := eos.Open(srcPath) if err != nil { return err } defer src.Close() dir := filepath.Dir(destPath) err = os.MkdirAll(dir, 0755) if err != nil { return err } flags := os.O_CREATE | os.O_WRONLY dest, err := os.OpenFile(destPath, flags, 0644) if err != nil { return err } defer dest.Close() stats, err := src.Stat() if err != nil { return err } totalBytes := int64(stats.Size()) startOffset := int64(0) if resume { startOffset, err = dest.Seek(0, os.SEEK_END) if err != nil { return err } if startOffset == 0 { comm.Logf("Downloading %s", humanize.IBytes(uint64(totalBytes))) } else if startOffset > totalBytes { comm.Logf("Existing data too big (%s > %s), starting over", humanize.IBytes(uint64(startOffset)), humanize.IBytes(uint64(totalBytes))) } else if startOffset == totalBytes { comm.Logf("All %s already there", humanize.IBytes(uint64(totalBytes))) return nil } comm.Logf("Resuming at %s / %s", humanize.IBytes(uint64(startOffset)), humanize.IBytes(uint64(totalBytes))) _, err = src.Seek(startOffset, os.SEEK_SET) if err != nil { return err } } else { comm.Logf("Downloading %s", humanize.IBytes(uint64(totalBytes))) } start := time.Now() comm.Progress(float64(startOffset) / float64(totalBytes)) comm.StartProgressWithTotalBytes(totalBytes) cw := counter.NewWriterCallback(func(count int64) { alpha := float64(startOffset+count) / float64(totalBytes) comm.Progress(alpha) }, dest) copiedBytes, err := io.Copy(cw, src) if err != nil { return err } comm.EndProgress() totalDuration := time.Since(start) prettyStartOffset := humanize.IBytes(uint64(startOffset)) prettySize := humanize.IBytes(uint64(copiedBytes)) perSecond := humanize.IBytes(uint64(float64(totalBytes-startOffset) / totalDuration.Seconds())) comm.Statf("%s + %s copied @ %s/s\n", prettyStartOffset, prettySize, perSecond) return nil }
func (f *FSCommand) Run(args []string) int { var verbose, machine, job, stat, tail, follow bool var numLines, numBytes int64 flags := f.Meta.FlagSet("fs", FlagSetClient) flags.Usage = func() { f.Ui.Output(f.Help()) } flags.BoolVar(&verbose, "verbose", false, "") flags.BoolVar(&machine, "H", false, "") flags.BoolVar(&job, "job", false, "") flags.BoolVar(&stat, "stat", false, "") flags.BoolVar(&follow, "f", false, "") flags.BoolVar(&tail, "tail", false, "") flags.Int64Var(&numLines, "n", -1, "") flags.Int64Var(&numBytes, "c", -1, "") if err := flags.Parse(args); err != nil { return 1 } args = flags.Args() if len(args) < 1 { if job { f.Ui.Error("job ID is required") } else { f.Ui.Error("allocation ID is required") } return 1 } if len(args) > 2 { f.Ui.Error(f.Help()) return 1 } path := "/" if len(args) == 2 { path = args[1] } client, err := f.Meta.Client() if err != nil { f.Ui.Error(fmt.Sprintf("Error initializing client: %v", err)) return 1 } // If -job is specified, use random allocation, otherwise use provided allocation allocID := args[0] if job { allocID, err = getRandomJobAlloc(client, args[0]) if err != nil { f.Ui.Error(fmt.Sprintf("Error fetching allocations: %v", err)) return 1 } } // Truncate the id unless full length is requested length := shortId if verbose { length = fullId } // Query the allocation info if len(allocID) == 1 { f.Ui.Error(fmt.Sprintf("Alloc ID must contain at least two characters.")) return 1 } if len(allocID)%2 == 1 { // Identifiers must be of even length, so we strip off the last byte // to provide a consistent user experience. allocID = allocID[:len(allocID)-1] } allocs, _, err := client.Allocations().PrefixList(allocID) if err != nil { f.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err)) return 1 } if len(allocs) == 0 { f.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID)) return 1 } if len(allocs) > 1 { // Format the allocs out := make([]string, len(allocs)+1) out[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status" for i, alloc := range allocs { out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s", limit(alloc.ID, length), limit(alloc.EvalID, length), alloc.JobID, alloc.TaskGroup, alloc.DesiredStatus, alloc.ClientStatus, ) } f.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", formatList(out))) return 0 } // Prefix lookup matched a single allocation alloc, _, err := client.Allocations().Info(allocs[0].ID, nil) if err != nil { f.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) return 1 } // Get file stat info file, _, err := client.AllocFS().Stat(alloc, path, nil) if err != nil { f.Ui.Error(err.Error()) return 1 } // If we want file stats, print those and exit. if stat { // Display the file information out := make([]string, 2) out[0] = "Mode|Size|Modified Time|Name" if file != nil { fn := file.Name if file.IsDir { fn = fmt.Sprintf("%s/", fn) } var size string if machine { size = fmt.Sprintf("%d", file.Size) } else { size = humanize.IBytes(uint64(file.Size)) } out[1] = fmt.Sprintf("%s|%s|%s|%s", file.FileMode, size, formatTime(file.ModTime), fn) } f.Ui.Output(formatList(out)) return 0 } // Determine if the path is a file or a directory. if file.IsDir { // We have a directory, list it. files, _, err := client.AllocFS().List(alloc, path, nil) if err != nil { f.Ui.Error(fmt.Sprintf("Error listing alloc dir: %s", err)) return 1 } // Display the file information in a tabular format out := make([]string, len(files)+1) out[0] = "Mode|Size|Modified Time|Name" for i, file := range files { fn := file.Name if file.IsDir { fn = fmt.Sprintf("%s/", fn) } var size string if machine { size = fmt.Sprintf("%d", file.Size) } else { size = humanize.IBytes(uint64(file.Size)) } out[i+1] = fmt.Sprintf("%s|%s|%s|%s", file.FileMode, size, formatTime(file.ModTime), fn, ) } f.Ui.Output(formatList(out)) return 0 } // We have a file, output it. var r io.ReadCloser var readErr error if !tail { if follow { r, readErr = f.followFile(client, alloc, path, api.OriginStart, 0, -1) } else { r, readErr = client.AllocFS().Cat(alloc, path, nil) } if readErr != nil { readErr = fmt.Errorf("Error reading file: %v", readErr) } } else { // Parse the offset var offset int64 = defaultTailLines * bytesToLines if nLines, nBytes := numLines != -1, numBytes != -1; nLines && nBytes { f.Ui.Error("Both -n and -c are not allowed") return 1 } else if numLines < -1 || numBytes < -1 { f.Ui.Error("Invalid size is specified") return 1 } else if nLines { offset = numLines * bytesToLines } else if nBytes { offset = numBytes } else { numLines = defaultTailLines } if offset > file.Size { offset = file.Size } if follow { r, readErr = f.followFile(client, alloc, path, api.OriginEnd, offset, numLines) } else { // This offset needs to be relative from the front versus the follow // is relative to the end offset = file.Size - offset r, readErr = client.AllocFS().ReadAt(alloc, path, offset, -1, nil) // If numLines is set, wrap the reader if numLines != -1 { r = NewLineLimitReader(r, int(numLines), int(numLines*bytesToLines), 1*time.Second) } } if readErr != nil { readErr = fmt.Errorf("Error tailing file: %v", readErr) } } defer r.Close() if readErr != nil { f.Ui.Error(readErr.Error()) return 1 } io.Copy(os.Stdout, r) return 0 }
return t.Format("2006/01/02 15:04:05 MST") }, "fmtip": func(addr net.Addr) string { switch addr := addr.(type) { case *net.TCPAddr: return addr.IP.String() case *net.UDPAddr: return addr.IP.String() default: return addr.String() } }, "fmtbytecount": func(v interface{}) (string, error) { var buint64 uint64 switch v := v.(type) { case uint64: buint64 = v case int64: buint64 = uint64(v) case float64: buint64 = uint64(v) default: return "", fmt.Errorf("unknown type for byte value %T", v) } return humanize.IBytes(buint64), nil }, "fmtNStoMS": func(nanos float64) float64 { return nanos / 1e6 }, }
func convert(value uint64) string { return humanize.IBytes(value) }
// Patch applies patch to old, according to the bspatch algorithm, // and writes the result to new. func Patch(old io.Reader, new io.Writer, newSize int64, readMessage ReadMessageFunc) error { // TODO: still debating whether we should take an io.ReadSeeker instead... probably? // the consumer can still do `ReadAll` themselves and pass a bytes.NewBuffer() obuf, err := ioutil.ReadAll(old) if err != nil { return err } // TODO: write directly to new instead of using a buffer nbuf := make([]byte, newSize) var oldpos, newpos int64 ctrl := &Control{} for { ctrl.Reset() err = readMessage(ctrl) if err != nil { return err } if ctrl.Eof { break } // Sanity-check if newpos+int64(len(ctrl.Add)) > newSize { return ErrCorrupt } // Add old data to diff string for i := int64(0); i < int64(len(ctrl.Add)); i++ { nbuf[newpos+i] = ctrl.Add[i] + obuf[oldpos+i] } // Adjust pointers newpos += int64(len(ctrl.Add)) oldpos += int64(len(ctrl.Add)) // Sanity-check if newpos+int64(len(ctrl.Copy)) > newSize { return ErrCorrupt } // Read extra string copy(nbuf[newpos:], ctrl.Copy) // Adjust pointers newpos += int64(len(ctrl.Copy)) oldpos += ctrl.Seek } if newpos != newSize { return fmt.Errorf("bsdiff: expected new file to be %d, was %d (%s difference)", newSize, newpos, humanize.IBytes(uint64(newSize-newpos))) } // Write the new file _, err = new.Write(nbuf) if err != nil { return err } return nil }
func doCmdBsdiff(target string, source string, patch string, concurrency int, measureOverhead bool) error { targetReader, err := os.Open(target) if err != nil { return err } defer targetReader.Close() targetStats, err := targetReader.Stat() if err != nil { return err } sourceReader, err := os.Open(source) if err != nil { return err } defer sourceReader.Close() sourceStats, err := sourceReader.Stat() if err != nil { return err } comm.Opf("Diffing %s (%s) and %s (%s)...", target, humanize.IBytes(uint64(targetStats.Size())), source, humanize.IBytes(uint64(sourceStats.Size()))) patchWriter, err := os.Create(patch) if err != nil { return err } wctx := wire.NewWriteContext(patchWriter) err = wctx.WriteMagic(pwr.PatchMagic) if err != nil { return err } compression := butlerCompressionSettings() err = wctx.WriteMessage(&pwr.PatchHeader{ Compression: &compression, }) if err != nil { return err } wctx, err = pwr.CompressWire(wctx, &compression) if err != nil { return err } targetContainer := &tlc.Container{} targetContainer.Files = []*tlc.File{ &tlc.File{ Path: target, Size: targetStats.Size(), }, } err = wctx.WriteMessage(targetContainer) if err != nil { return err } sourceContainer := &tlc.Container{} sourceContainer.Files = []*tlc.File{ &tlc.File{ Path: source, Size: sourceStats.Size(), }, } err = wctx.WriteMessage(sourceContainer) if err != nil { return err } err = wctx.WriteMessage(&pwr.SyncHeader{ FileIndex: 0, }) if err != nil { return err } err = wctx.WriteMessage(&pwr.SyncOp{ Type: pwr.SyncOp_BSDIFF, FileIndex: 0, }) if err != nil { return err } startTime := time.Now() comm.StartProgress() dc := bsdiff.DiffContext{ MeasureMem: *appArgs.memstats, MeasureParallelOverhead: measureOverhead, SuffixSortConcurrency: concurrency, } err = dc.Do(targetReader, sourceReader, wctx.WriteMessage, comm.NewStateConsumer()) if err != nil { return err } comm.EndProgress() err = wctx.WriteMessage(&pwr.SyncOp{ Type: pwr.SyncOp_HEY_YOU_DID_IT, }) if err != nil { return err } err = wctx.Close() if err != nil { return err } patchStats, err := os.Lstat(patch) if err != nil { return err } duration := time.Since(startTime) perSec := float64(sourceStats.Size()) / duration.Seconds() relToNew := 100.0 * float64(patchStats.Size()) / float64(sourceStats.Size()) comm.Statf("Processed %s @ %s / s, total %s", humanize.IBytes(uint64(sourceStats.Size())), humanize.IBytes(uint64(perSec)), duration) comm.Statf("Wrote %s patch (%.2f%% of total size) to %s", humanize.IBytes(uint64(patchStats.Size())), relToNew, patch) return nil }
func (f *FSCommand) Run(args []string) int { var verbose, machine, job, stat bool flags := f.Meta.FlagSet("fs-list", FlagSetClient) flags.Usage = func() { f.Ui.Output(f.Help()) } flags.BoolVar(&verbose, "verbose", false, "") flags.BoolVar(&machine, "H", false, "") flags.BoolVar(&job, "job", false, "") flags.BoolVar(&stat, "stat", false, "") if err := flags.Parse(args); err != nil { return 1 } args = flags.Args() if len(args) < 1 { if job { f.Ui.Error("job ID is required") } else { f.Ui.Error("allocation ID is required") } return 1 } path := "/" if len(args) == 2 { path = args[1] } client, err := f.Meta.Client() if err != nil { f.Ui.Error(fmt.Sprintf("Error initializing client: %v", err)) return 1 } // If -job is specified, use random allocation, otherwise use provided allocation allocID := args[0] if job { allocID, err = getRandomJobAlloc(client, args[0]) if err != nil { f.Ui.Error(fmt.Sprintf("Error fetching allocations: %v", err)) return 1 } } // Truncate the id unless full length is requested length := shortId if verbose { length = fullId } // Query the allocation info if len(allocID) == 1 { f.Ui.Error(fmt.Sprintf("Alloc ID must contain at least two characters.")) return 1 } if len(allocID)%2 == 1 { // Identifiers must be of even length, so we strip off the last byte // to provide a consistent user experience. allocID = allocID[:len(allocID)-1] } allocs, _, err := client.Allocations().PrefixList(allocID) if err != nil { f.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err)) return 1 } if len(allocs) == 0 { f.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID)) return 1 } if len(allocs) > 1 { // Format the allocs out := make([]string, len(allocs)+1) out[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status" for i, alloc := range allocs { out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s", limit(alloc.ID, length), limit(alloc.EvalID, length), alloc.JobID, alloc.TaskGroup, alloc.DesiredStatus, alloc.ClientStatus, ) } f.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", formatList(out))) return 0 } // Prefix lookup matched a single allocation alloc, _, err := client.Allocations().Info(allocs[0].ID, nil) if err != nil { f.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) return 1 } if alloc.DesiredStatus == "failed" { allocID := limit(alloc.ID, length) msg := fmt.Sprintf(`The allocation %q failed to be placed. To see the cause, run: nomad alloc-status %s`, allocID, allocID) f.Ui.Error(msg) return 0 } // Get file stat info file, _, err := client.AllocFS().Stat(alloc, path, nil) if err != nil { f.Ui.Error(err.Error()) return 1 } // If we want file stats, print those and exit. if stat { // Display the file information out := make([]string, 2) out[0] = "Mode|Size|Modified Time|Name" if file != nil { fn := file.Name if file.IsDir { fn = fmt.Sprintf("%s/", fn) } var size string if machine { size = fmt.Sprintf("%d", file.Size) } else { size = humanize.IBytes(uint64(file.Size)) } out[1] = fmt.Sprintf("%s|%s|%s|%s", file.FileMode, size, formatTime(file.ModTime), fn) } f.Ui.Output(formatList(out)) return 0 } // Determine if the path is a file or a directory. if file.IsDir { // We have a directory, list it. files, _, err := client.AllocFS().List(alloc, path, nil) if err != nil { f.Ui.Error(fmt.Sprintf("Error listing alloc dir: %s", err)) return 1 } // Display the file information in a tabular format out := make([]string, len(files)+1) out[0] = "Mode|Size|Modified Time|Name" for i, file := range files { fn := file.Name if file.IsDir { fn = fmt.Sprintf("%s/", fn) } var size string if machine { size = fmt.Sprintf("%d", file.Size) } else { size = humanize.IBytes(uint64(file.Size)) } out[i+1] = fmt.Sprintf("%s|%s|%s|%s", file.FileMode, size, formatTime(file.ModTime), fn, ) } f.Ui.Output(formatList(out)) } else { // We have a file, cat it. r, _, err := client.AllocFS().Cat(alloc, path, nil) if err != nil { f.Ui.Error(fmt.Sprintf("Error reading file: %s", err)) return 1 } io.Copy(os.Stdout, r) } return 0 }