func main() { // Actually the delimiter uses runes, which can be multiple characters long. // https://blog.golang.org/strings delimiter := flag.String("delimiter", ",", "field delimiter for csv file, must be exactly one character long.") spec.RegisterDatabaseFlags(flag.CommandLine) profile.RegisterProfileFlags(flag.CommandLine) flag.Usage = func() { fmt.Fprintln(os.Stderr, "Usage: csv-export [options] dataset > filename") flag.PrintDefaults() } flag.Parse(true) if flag.NArg() != 1 { d.CheckError(errors.New("expected dataset arg")) } ds, err := spec.GetDataset(flag.Arg(0)) d.CheckError(err) defer ds.Database().Close() comma, err := csv.StringToRune(*delimiter) d.CheckError(err) err = d.Try(func() { defer profile.MaybeStartProfile().Stop() nomsList, structDesc := csv.ValueToListAndElemDesc(ds.HeadValue(), ds.Database()) csv.Write(nomsList, structDesc, comma, os.Stdout) }) if err != nil { fmt.Println("Failed to export dataset as CSV:") fmt.Println(err) } }
func main() { err := d.Try(func() { spec.RegisterDatabaseFlags(flag.CommandLine) profile.RegisterProfileFlags(flag.CommandLine) flag.Usage = customUsage flag.Parse(true) if flag.NArg() != 2 { d.CheckError(errors.New("Expected directory path followed by dataset")) } dir := flag.Arg(0) ds, err := spec.GetDataset(flag.Arg(1)) d.CheckError(err) defer profile.MaybeStartProfile().Stop() cpuCount := runtime.NumCPU() filesChan := make(chan fileIndex, 1024) refsChan := make(chan refIndex, 1024) getFilePaths := func() { index := 0 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { d.PanicIfTrue(err != nil, "Cannot traverse directories") if !info.IsDir() && filepath.Ext(path) == ".xml" { filesChan <- fileIndex{path, index} index++ } return nil }) d.PanicIfError(err) close(filesChan) } wg := sync.WaitGroup{} importXML := func() { expectedType := types.NewMap() for f := range filesChan { file, err := os.Open(f.path) d.PanicIfTrue(err != nil, "Error getting XML") xmlObject, err := mxj.NewMapXmlReader(file) d.PanicIfTrue(err != nil, "Error decoding XML") object := xmlObject.Old() file.Close() nomsObj := jsontonoms.NomsValueFromDecodedJSON(object, false) d.Chk.IsType(expectedType, nomsObj) var r types.Ref if !*noIO { r = ds.Database().WriteValue(nomsObj) } refsChan <- refIndex{r, f.index} } wg.Done() } go getFilePaths() for i := 0; i < cpuCount*8; i++ { wg.Add(1) go importXML() } go func() { wg.Wait() close(refsChan) // done converting xml to noms }() refList := refIndexList{} for r := range refsChan { refList = append(refList, r) } sort.Sort(refList) refs := make([]types.Value, len(refList)) for idx, r := range refList { refs[idx] = r.ref } rl := types.NewList(refs...) if !*noIO { _, err := ds.CommitValue(rl) d.PanicIfError(err) } }) if err != nil { log.Fatal(err) } }
func main() { profile.RegisterProfileFlags(flag.CommandLine) flag.Parse(true) buildCount := *count insertCount := buildCount / 50 defer profile.MaybeStartProfile().Stop() collectionTypes := []string{"List", "Set", "Map"} buildFns := []buildCollectionFn{buildList, buildSet, buildMap} buildIncrFns := []buildCollectionFn{buildListIncrementally, buildSetIncrementally, buildMapIncrementally} readFns := []readCollectionFn{readList, readSet, readMap} elementTypes := []string{"numbers (8 B)", "strings (32 B)", "structs (64 B)"} elementSizes := []uint64{numberSize, stringSize, structSize} valueFns := []createValueFn{createNumber, createString, createStruct} for i, colType := range collectionTypes { fmt.Printf("Testing %s: \t\tbuild %d\t\t\tscan %d\t\t\tinsert %d\n", colType, buildCount, buildCount, insertCount) for j, elementType := range elementTypes { valueFn := valueFns[j] // Build One-Time ms := chunks.NewMemoryStore() ds := dataset.NewDataset(datas.NewDatabase(ms), "test") t1 := time.Now() col := buildFns[i](buildCount, valueFn) ds, err := ds.CommitValue(col) d.Chk.NoError(err) buildDuration := time.Since(t1) // Read t1 = time.Now() col = ds.HeadValue().(types.Collection) readFns[i](col) readDuration := time.Since(t1) // Build Incrementally ms = chunks.NewMemoryStore() ds = dataset.NewDataset(datas.NewDatabase(ms), "test") t1 = time.Now() col = buildIncrFns[i](insertCount, valueFn) ds, err = ds.CommitValue(col) d.Chk.NoError(err) incrDuration := time.Since(t1) elementSize := elementSizes[j] buildSize := elementSize * buildCount incrSize := elementSize * insertCount fmt.Printf("%s\t\t%s\t\t%s\t\t%s\n", elementType, rate(buildDuration, buildSize), rate(readDuration, buildSize), rate(incrDuration, incrSize)) } fmt.Println() } fmt.Printf("Testing Blob: \t\tbuild %d MB\t\t\tscan %d MB\n", *blobSize/1000000, *blobSize/1000000) ms := chunks.NewMemoryStore() ds := dataset.NewDataset(datas.NewDatabase(ms), "test") blobBytes := makeBlobBytes(*blobSize) t1 := time.Now() blob := types.NewBlob(bytes.NewReader(blobBytes)) ds.CommitValue(blob) buildDuration := time.Since(t1) ds = dataset.NewDataset(datas.NewDatabase(ms), "test") t1 = time.Now() blob = ds.HeadValue().(types.Blob) outBytes, _ := ioutil.ReadAll(blob.Reader()) readDuration := time.Since(t1) d.Chk.True(bytes.Compare(blobBytes, outBytes) == 0) fmt.Printf("\t\t\t%s\t\t%s\n\n", rate(buildDuration, *blobSize), rate(readDuration, *blobSize)) }
func main() { // Actually the delimiter uses runes, which can be multiple characters long. // https://blog.golang.org/strings delimiter := flag.String("delimiter", ",", "field delimiter for csv file, must be exactly one character long.") comment := flag.String("comment", "", "comment to add to commit's meta data") header := flag.String("header", "", "header row. If empty, we'll use the first row of the file") name := flag.String("name", "Row", "struct name. The user-visible name to give to the struct type that will hold each row of data.") columnTypes := flag.String("column-types", "", "a comma-separated list of types representing the desired type of each column. if absent all types default to be String") pathDescription := "noms path to blob to import" path := flag.String("path", "", pathDescription) flag.StringVar(path, "p", "", pathDescription) dateFlag := flag.String("date", "", fmt.Sprintf(`date of commit in ISO 8601 format ("%s"). By default, the current date is used.`, dateFormat)) noProgress := flag.Bool("no-progress", false, "prevents progress from being output if true") destType := flag.String("dest-type", "list", "the destination type to import to. can be 'list' or 'map:<pk>', where <pk> is the index position (0-based) of the column that is a the unique identifier for the column") skipRecords := flag.Uint("skip-records", 0, "number of records to skip at beginning of file") destTypePattern := regexp.MustCompile("^(list|map):(\\d+)$") spec.RegisterDatabaseFlags(flag.CommandLine) profile.RegisterProfileFlags(flag.CommandLine) flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: csv-import [options] <csvfile> <dataset>\n\n") flag.PrintDefaults() } flag.Parse(true) var err error switch { case flag.NArg() == 0: err = errors.New("Maybe you put options after the dataset?") case flag.NArg() == 1 && *path == "": err = errors.New("If <csvfile> isn't specified, you must specify a noms path with -p") case flag.NArg() == 2 && *path != "": err = errors.New("Cannot specify both <csvfile> and a noms path with -p") case flag.NArg() > 2: err = errors.New("Too many arguments") } d.CheckError(err) var date = *dateFlag if date == "" { date = time.Now().UTC().Format(dateFormat) } else { _, err := time.Parse(dateFormat, date) d.CheckErrorNoUsage(err) } defer profile.MaybeStartProfile().Stop() var r io.Reader var size uint64 var filePath string var dataSetArgN int if *path != "" { db, val, err := spec.GetPath(*path) d.CheckError(err) if val == nil { d.CheckError(fmt.Errorf("Path %s not found\n", *path)) } blob, ok := val.(types.Blob) if !ok { d.CheckError(fmt.Errorf("Path %s not a Blob: %s\n", *path, types.EncodedValue(val.Type()))) } defer db.Close() r = blob.Reader() size = blob.Len() dataSetArgN = 0 } else { filePath = flag.Arg(0) res, err := os.Open(filePath) d.CheckError(err) defer res.Close() fi, err := res.Stat() d.CheckError(err) r = res size = uint64(fi.Size()) dataSetArgN = 1 } if !*noProgress { r = progressreader.New(r, getStatusPrinter(size)) } comma, err := csv.StringToRune(*delimiter) d.CheckErrorNoUsage(err) var dest int var pk int if *destType == "list" { dest = destList } else if match := destTypePattern.FindStringSubmatch(*destType); match != nil { dest = destMap pk, err = strconv.Atoi(match[2]) d.CheckErrorNoUsage(err) } else { fmt.Println("Invalid dest-type: ", *destType) return } cr := csv.NewCSVReader(r, comma) for i := uint(0); i < *skipRecords; i++ { cr.Read() } var headers []string if *header == "" { headers, err = cr.Read() d.PanicIfError(err) } else { headers = strings.Split(*header, string(comma)) } ds, err := spec.GetDataset(flag.Arg(dataSetArgN)) d.CheckError(err) defer ds.Database().Close() kinds := []types.NomsKind{} if *columnTypes != "" { kinds = csv.StringsToKinds(strings.Split(*columnTypes, ",")) } var value types.Value if dest == destList { value, _ = csv.ReadToList(cr, *name, headers, kinds, ds.Database()) } else { value = csv.ReadToMap(cr, headers, pk, kinds, ds.Database()) } mi := metaInfoForCommit(date, filePath, *path, *comment) _, err = ds.Commit(value, dataset.CommitOptions{Meta: mi}) if !*noProgress { status.Clear() } d.PanicIfError(err) }