func CheckError(err error) { if err != nil { fmt.Fprintf(os.Stderr, "error: %s\n", err) flag.Usage() UtilExiter.Exit(-1) } }
func main() { useSHA := flag.String("use-sha", "", "<default>=no hashing, 1=sha1, 256=sha256, 512=sha512, blake=blake2b") useBH := flag.Bool("use-bh", false, "whether we buzhash the bytes") flag.Parse(true) flag.Usage = func() { fmt.Printf("%s <big-file>\n", os.Args[0]) flag.PrintDefaults() return } if len(flag.Args()) < 1 { flag.Usage() return } p := flag.Args()[0] bh := buzhash.NewBuzHash(64 * 8) f, _ := os.Open(p) defer f.Close() t0 := time.Now() buf := make([]byte, 4*1024) l := uint64(0) var h hash.Hash if *useSHA == "1" { h = sha1.New() } else if *useSHA == "256" { h = sha256.New() } else if *useSHA == "512" { h = sha512.New() } else if *useSHA == "blake" { h = blake2.NewBlake2B() } for { n, err := f.Read(buf) l += uint64(n) if err == io.EOF { break } s := buf[:n] if h != nil { h.Write(s) } if *useBH { bh.Write(s) } } t1 := time.Now() d := t1.Sub(t0) fmt.Printf("Read %s in %s (%s/s)\n", humanize.Bytes(l), d, humanize.Bytes(uint64(float64(l)/d.Seconds()))) digest := []byte{} if h != nil { fmt.Printf("%x\n", h.Sum(digest)) } }
func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s <url> <dataset>\n", os.Args[0]) flag.PrintDefaults() } spec.RegisterDatabaseFlags(flag.CommandLine) flag.Parse(true) if len(flag.Args()) != 2 { d.CheckError(errors.New("expected url and dataset flags")) } ds, err := spec.GetDataset(flag.Arg(1)) d.CheckError(err) url := flag.Arg(0) if url == "" { flag.Usage() } res, err := http.Get(url) if err != nil { log.Fatalf("Error fetching %s: %+v\n", url, err) } else if res.StatusCode != 200 { log.Fatalf("Error fetching %s: %s\n", url, res.Status) } defer res.Body.Close() var jsonObject interface{} err = json.NewDecoder(res.Body).Decode(&jsonObject) if err != nil { log.Fatalln("Error decoding JSON: ", err) } _, err = ds.CommitValue(jsontonoms.NomsValueFromDecodedJSON(jsonObject, true)) d.PanicIfError(err) ds.Database().Close() }
func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s:\n %s path/to/staging/dir\n", os.Args[0], os.Args[0]) } flag.Parse(true) if flag.Arg(0) == "" { flag.Usage() os.Exit(1) } err := d.Try(func() { stagingDir, err := filepath.Abs(flag.Arg(0)) d.PanicIfTrue(err != nil, "Path to staging directory (first arg) must be valid, not %s", flag.Arg(0)) d.PanicIfError(os.MkdirAll(stagingDir, 0755)) goPath := os.Getenv("GOPATH") d.PanicIfTrue(goPath == "", "GOPATH must be set!") workspace := os.Getenv("WORKSPACE") if workspace == "" { fmt.Printf("WORKSPACE not set in environment; using GOPATH (%s).\n", goPath) workspace = goPath } pythonPath := filepath.Join(goPath, nomsCheckoutPath, "tools") env := runner.Env{ "PYTHONPATH": pythonPath, } if !runner.Serial(os.Stdout, os.Stderr, env, ".", buildScript) { os.Exit(1) } if !runner.Serial(os.Stdout, os.Stderr, env, ".", stageScript, stagingDir) { os.Exit(1) } }) if err != nil { log.Fatal(err) } }
func main() { comment := flag.String("comment", "", "comment to add to commit's meta data") stdin := flag.Bool("stdin", false, "read blob from stdin") spec.RegisterDatabaseFlags(flag.CommandLine) flag.Usage = func() { fmt.Fprintf(os.Stderr, "Fetches a URL, file, or stdin into a noms blob\n\nUsage: %s [--stdin?] [url-or-local-path?] [dataset]\n", os.Args[0]) flag.PrintDefaults() } flag.Parse(true) if !(*stdin && flag.NArg() == 1) && flag.NArg() != 2 { flag.Usage() os.Exit(-1) } start = time.Now() ds, err := spec.GetDataset(flag.Arg(flag.NArg() - 1)) d.CheckErrorNoUsage(err) defer ds.Database().Close() var r io.Reader var contentLength int64 var sourceType, sourceVal string if *stdin { r = os.Stdin contentLength = -1 } else if url := flag.Arg(0); strings.HasPrefix(url, "http") { resp, err := http.Get(url) if err != nil { fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %s\n", url, err) return } switch resp.StatusCode / 100 { case 4, 5: fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %d (%s)\n", url, resp.StatusCode, resp.Status) return } r = resp.Body contentLength = resp.ContentLength sourceType, sourceVal = "url", url } else { // assume it's a file f, err := os.Open(url) if err != nil { fmt.Fprintf(os.Stderr, "Invalid URL %s - does not start with 'http' and isn't local file either. fopen error: %s", url, err) return } s, err := f.Stat() if err != nil { fmt.Fprintf(os.Stderr, "Could not stat file %s: %s", url, err) return } r = f contentLength = s.Size() sourceType, sourceVal = "file", url } pr := progressreader.New(r, getStatusPrinter(contentLength)) b := types.NewStreamingBlob(pr, ds.Database()) mi := metaInfoForCommit(sourceType, sourceVal, *comment) ds, err = ds.Commit(b, dataset.CommitOptions{Meta: mi}) if err != nil { d.Chk.Equal(datas.ErrMergeNeeded, err) fmt.Fprintf(os.Stderr, "Could not commit, optimistic concurrency failed.") return } status.Done() fmt.Println("Done") }