func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s <dataset> <file>\n", os.Args[0]) flag.PrintDefaults() } spec.RegisterDatabaseFlags(flag.CommandLine) flag.Parse(true) if len(flag.Args()) != 2 { d.CheckError(errors.New("expected dataset and file flags")) } var blob types.Blob path := flag.Arg(0) if db, val, err := spec.GetPath(path); err != nil { d.CheckErrorNoUsage(err) } else if val == nil { d.CheckErrorNoUsage(fmt.Errorf("No value at %s", path)) } else if b, ok := val.(types.Blob); !ok { d.CheckErrorNoUsage(fmt.Errorf("Value at %s is not a blob", path)) } else { defer db.Close() blob = b } filePath := flag.Arg(1) if filePath == "" { d.CheckErrorNoUsage(errors.New("Empty file path")) } // Note: overwrites any existing file. file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0644) d.CheckErrorNoUsage(err) defer file.Close() expected := humanize.Bytes(blob.Len()) start := time.Now() progReader := progressreader.New(blob.Reader(), func(seen uint64) { elapsed := time.Since(start).Seconds() rate := uint64(float64(seen) / elapsed) status.Printf("%s of %s written in %ds (%s/s)...", humanize.Bytes(seen), expected, int(elapsed), humanize.Bytes(rate)) }) io.Copy(file, progReader) status.Done() }
func main() { var ( // Actually the delimiter uses runes, which can be multiple characters long. // https://blog.golang.org/strings delimiter = flag.String("delimiter", ",", "field delimiter for csv file, must be exactly one character long.") header = flag.String("header", "", "header row. If empty, we'll use the first row of the file") name = flag.String("name", "Row", "struct name. The user-visible name to give to the struct type that will hold each row of data.") columnTypes = flag.String("column-types", "", "a comma-separated list of types representing the desired type of each column. if absent all types default to be String") noProgress = flag.Bool("no-progress", false, "prevents progress from being output if true") destType = flag.String("dest-type", "list", "the destination type to import to. can be 'list' or 'map:<pk>', where <pk> is the index position (0-based) of the column that is a the unique identifier for the column") destTypePattern = regexp.MustCompile("^(list|map):(\\d+)$") ) spec.RegisterDatabaseFlags() cpuCount := runtime.NumCPU() runtime.GOMAXPROCS(cpuCount) flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: csv-import [options] <dataset> <csvfile>\n\n") flag.PrintDefaults() } flag.Parse() if flag.NArg() != 2 { err := fmt.Errorf("Expected exactly two parameters (dataset and path) after flags, but you have %d. Maybe you put a flag after the path?", flag.NArg()) util.CheckError(err) } path := flag.Arg(1) defer profile.MaybeStartProfile().Stop() res, err := os.Open(path) d.Exp.NoError(err) defer res.Close() comma, err := csv.StringToRune(*delimiter) if err != nil { util.CheckError(err) return } var dest int var pk int if *destType == "list" { dest = destList } else if match := destTypePattern.FindStringSubmatch(*destType); match != nil { dest = destMap pk, err = strconv.Atoi(match[2]) d.Chk.NoError(err) } else { fmt.Println("Invalid dest-type: ", *destType) return } fi, err := res.Stat() d.Chk.NoError(err) var r io.Reader = res if !*noProgress { r = progressreader.New(r, getStatusPrinter(uint64(fi.Size()))) } cr := csv.NewCSVReader(r, comma) var headers []string if *header == "" { headers, err = cr.Read() d.Exp.NoError(err) } else { headers = strings.Split(*header, string(comma)) } ds, err := spec.GetDataset(flag.Arg(0)) util.CheckError(err) defer ds.Database().Close() kinds := []types.NomsKind{} if *columnTypes != "" { kinds = csv.StringsToKinds(strings.Split(*columnTypes, ",")) } var value types.Value if dest == destList { value, _ = csv.ReadToList(cr, *name, headers, kinds, ds.Database()) } else { value = csv.ReadToMap(cr, headers, pk, kinds, ds.Database()) } _, err = ds.Commit(value) if !*noProgress { status.Clear() } d.Exp.NoError(err) }
func main() { comment := flag.String("comment", "", "comment to add to commit's meta data") spec.RegisterDatabaseFlags(flag.CommandLine) flag.Usage = func() { fmt.Fprintf(os.Stderr, "Fetches a URL (or file) into a noms blob\n\nUsage: %s <url-or-local-path> <dataset>:\n", os.Args[0]) flag.PrintDefaults() } flag.Parse(true) if flag.NArg() != 2 { d.CheckErrorNoUsage(errors.New("expected dataset and url arguments")) } ds, err := spec.GetDataset(flag.Arg(1)) d.CheckErrorNoUsage(err) defer ds.Database().Close() url := flag.Arg(0) fileOrUrl := "file" start = time.Now() var pr io.Reader if strings.HasPrefix(url, "http") { resp, err := http.Get(url) if err != nil { fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %s\n", url, err) return } switch resp.StatusCode / 100 { case 4, 5: fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %d (%s)\n", url, resp.StatusCode, resp.Status) return } pr = progressreader.New(resp.Body, getStatusPrinter(resp.ContentLength)) fileOrUrl = "url" } else { // assume it's a file f, err := os.Open(url) if err != nil { fmt.Fprintf(os.Stderr, "Invalid URL %s - does not start with 'http' and isn't local file either. fopen error: %s", url, err) return } s, err := f.Stat() if err != nil { fmt.Fprintf(os.Stderr, "Could not stat file %s: %s", url, err) return } pr = progressreader.New(f, getStatusPrinter(s.Size())) fileOrUrl = "file" } b := types.NewStreamingBlob(pr, ds.Database()) mi := metaInfoForCommit(fileOrUrl, url, *comment) ds, err = ds.Commit(b, dataset.CommitOptions{Meta: mi}) if err != nil { d.Chk.Equal(datas.ErrMergeNeeded, err) fmt.Fprintf(os.Stderr, "Could not commit, optimistic concurrency failed.") return } status.Done() fmt.Println("Done") }
func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "Fetches a URL into a noms blob\n\nUsage: %s <dataset> <url>:\n", os.Args[0]) flag.PrintDefaults() } spec.RegisterDatabaseFlags() flag.Parse() if flag.NArg() != 2 { util.CheckError(errors.New("expected dataset and url arguments")) } ds, err := spec.GetDataset(flag.Arg(0)) util.CheckError(err) defer ds.Database().Close() url := flag.Arg(1) start = time.Now() var pr io.Reader if strings.HasPrefix(url, "http") { resp, err := http.Get(url) if err != nil { fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %s\n", url, err) return } switch resp.StatusCode / 100 { case 4, 5: fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %d (%s)\n", url, resp.StatusCode, resp.Status) return } pr = progressreader.New(resp.Body, getStatusPrinter(resp.ContentLength)) } else { // assume it's a file f, err := os.Open(url) if err != nil { fmt.Fprintf(os.Stderr, "Invalid URL %s - does not start with 'http' and isn't local file either. fopen error: %s", url, err) return } s, err := f.Stat() if err != nil { fmt.Fprintf(os.Stderr, "Could not stat file %s: %s", url, err) return } pr = progressreader.New(f, getStatusPrinter(s.Size())) } b := types.NewBlob(pr) ds, err = ds.Commit(b) if err != nil { d.Chk.True(datas.ErrMergeNeeded == err) fmt.Fprintf(os.Stderr, "Could not commit, optimistic concurrency failed.") return } status.Done() fmt.Println("Done") }
func main() { // Actually the delimiter uses runes, which can be multiple characters long. // https://blog.golang.org/strings delimiter := flag.String("delimiter", ",", "field delimiter for csv file, must be exactly one character long.") comment := flag.String("comment", "", "comment to add to commit's meta data") header := flag.String("header", "", "header row. If empty, we'll use the first row of the file") name := flag.String("name", "Row", "struct name. The user-visible name to give to the struct type that will hold each row of data.") columnTypes := flag.String("column-types", "", "a comma-separated list of types representing the desired type of each column. if absent all types default to be String") pathDescription := "noms path to blob to import" path := flag.String("path", "", pathDescription) flag.StringVar(path, "p", "", pathDescription) dateFlag := flag.String("date", "", fmt.Sprintf(`date of commit in ISO 8601 format ("%s"). By default, the current date is used.`, dateFormat)) noProgress := flag.Bool("no-progress", false, "prevents progress from being output if true") destType := flag.String("dest-type", "list", "the destination type to import to. can be 'list' or 'map:<pk>', where <pk> is the index position (0-based) of the column that is a the unique identifier for the column") skipRecords := flag.Uint("skip-records", 0, "number of records to skip at beginning of file") destTypePattern := regexp.MustCompile("^(list|map):(\\d+)$") spec.RegisterDatabaseFlags(flag.CommandLine) profile.RegisterProfileFlags(flag.CommandLine) flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: csv-import [options] <csvfile> <dataset>\n\n") flag.PrintDefaults() } flag.Parse(true) var err error switch { case flag.NArg() == 0: err = errors.New("Maybe you put options after the dataset?") case flag.NArg() == 1 && *path == "": err = errors.New("If <csvfile> isn't specified, you must specify a noms path with -p") case flag.NArg() == 2 && *path != "": err = errors.New("Cannot specify both <csvfile> and a noms path with -p") case flag.NArg() > 2: err = errors.New("Too many arguments") } d.CheckError(err) var date = *dateFlag if date == "" { date = time.Now().UTC().Format(dateFormat) } else { _, err := time.Parse(dateFormat, date) d.CheckErrorNoUsage(err) } defer profile.MaybeStartProfile().Stop() var r io.Reader var size uint64 var filePath string var dataSetArgN int if *path != "" { db, val, err := spec.GetPath(*path) d.CheckError(err) if val == nil { d.CheckError(fmt.Errorf("Path %s not found\n", *path)) } blob, ok := val.(types.Blob) if !ok { d.CheckError(fmt.Errorf("Path %s not a Blob: %s\n", *path, types.EncodedValue(val.Type()))) } defer db.Close() r = blob.Reader() size = blob.Len() dataSetArgN = 0 } else { filePath = flag.Arg(0) res, err := os.Open(filePath) d.CheckError(err) defer res.Close() fi, err := res.Stat() d.CheckError(err) r = res size = uint64(fi.Size()) dataSetArgN = 1 } if !*noProgress { r = progressreader.New(r, getStatusPrinter(size)) } comma, err := csv.StringToRune(*delimiter) d.CheckErrorNoUsage(err) var dest int var pk int if *destType == "list" { dest = destList } else if match := destTypePattern.FindStringSubmatch(*destType); match != nil { dest = destMap pk, err = strconv.Atoi(match[2]) d.CheckErrorNoUsage(err) } else { fmt.Println("Invalid dest-type: ", *destType) return } cr := csv.NewCSVReader(r, comma) for i := uint(0); i < *skipRecords; i++ { cr.Read() } var headers []string if *header == "" { headers, err = cr.Read() d.PanicIfError(err) } else { headers = strings.Split(*header, string(comma)) } ds, err := spec.GetDataset(flag.Arg(dataSetArgN)) d.CheckError(err) defer ds.Database().Close() kinds := []types.NomsKind{} if *columnTypes != "" { kinds = csv.StringsToKinds(strings.Split(*columnTypes, ",")) } var value types.Value if dest == destList { value, _ = csv.ReadToList(cr, *name, headers, kinds, ds.Database()) } else { value = csv.ReadToMap(cr, headers, pk, kinds, ds.Database()) } mi := metaInfoForCommit(date, filePath, *path, *comment) _, err = ds.Commit(value, dataset.CommitOptions{Meta: mi}) if !*noProgress { status.Clear() } d.PanicIfError(err) }
func main() { comment := flag.String("comment", "", "comment to add to commit's meta data") stdin := flag.Bool("stdin", false, "read blob from stdin") spec.RegisterDatabaseFlags(flag.CommandLine) flag.Usage = func() { fmt.Fprintf(os.Stderr, "Fetches a URL, file, or stdin into a noms blob\n\nUsage: %s [--stdin?] [url-or-local-path?] [dataset]\n", os.Args[0]) flag.PrintDefaults() } flag.Parse(true) if !(*stdin && flag.NArg() == 1) && flag.NArg() != 2 { flag.Usage() os.Exit(-1) } start = time.Now() ds, err := spec.GetDataset(flag.Arg(flag.NArg() - 1)) d.CheckErrorNoUsage(err) defer ds.Database().Close() var r io.Reader var contentLength int64 var sourceType, sourceVal string if *stdin { r = os.Stdin contentLength = -1 } else if url := flag.Arg(0); strings.HasPrefix(url, "http") { resp, err := http.Get(url) if err != nil { fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %s\n", url, err) return } switch resp.StatusCode / 100 { case 4, 5: fmt.Fprintf(os.Stderr, "Could not fetch url %s, error: %d (%s)\n", url, resp.StatusCode, resp.Status) return } r = resp.Body contentLength = resp.ContentLength sourceType, sourceVal = "url", url } else { // assume it's a file f, err := os.Open(url) if err != nil { fmt.Fprintf(os.Stderr, "Invalid URL %s - does not start with 'http' and isn't local file either. fopen error: %s", url, err) return } s, err := f.Stat() if err != nil { fmt.Fprintf(os.Stderr, "Could not stat file %s: %s", url, err) return } r = f contentLength = s.Size() sourceType, sourceVal = "file", url } pr := progressreader.New(r, getStatusPrinter(contentLength)) b := types.NewStreamingBlob(pr, ds.Database()) mi := metaInfoForCommit(sourceType, sourceVal, *comment) ds, err = ds.Commit(b, dataset.CommitOptions{Meta: mi}) if err != nil { d.Chk.Equal(datas.ErrMergeNeeded, err) fmt.Fprintf(os.Stderr, "Could not commit, optimistic concurrency failed.") return } status.Done() fmt.Println("Done") }