func main() { runtime.GOMAXPROCS(runtime.NumCPU()) flags := datas.NewFlags() flag.Parse() ds, ok := flags.CreateDataStore() if !ok || *inputID == "" || *outputID == "" { flag.Usage() return } defer ds.Close() err := d.Try(func() { if util.MaybeStartCPUProfile() { defer util.StopCPUProfile() } inputDataset := dataset.NewDataset(ds, *inputID) outputDataset := dataset.NewDataset(ds, *outputID) input := inputDataset.Head().Value().(ListOfRefOfMapOfStringToValue) output := getIndex(input, ds) _, err := outputDataset.Commit(output) d.Exp.NoError(err) util.MaybeWriteMemProfile() }) if err != nil { log.Fatal(err) } }
func main() { cpuCount := runtime.NumCPU() runtime.GOMAXPROCS(cpuCount) flag.Parse() sourceStore, ok := sourceStoreFlags.CreateDataStore() sink := sinkDsFlags.CreateDataset() if !ok || sink == nil || *p == 0 || *sourceObject == "" { flag.Usage() return } defer sourceStore.Close() defer sink.Store().Close() err := d.Try(func() { if util.MaybeStartCPUProfile() { defer util.StopCPUProfile() } sourceRef := ref.Ref{} if r, ok := ref.MaybeParse(*sourceObject); ok { if sourceStore.Has(r) { sourceRef = r } } else { if c, ok := sourceStore.MaybeHead(*sourceObject); ok { sourceRef = c.Ref() } } d.Exp.False(sourceRef.IsEmpty(), "Unknown source object: %s", *sourceObject) var err error *sink, err = sink.Pull(sourceStore, sourceRef, int(*p)) util.MaybeWriteMemProfile() d.Exp.NoError(err) }) if err != nil { log.Fatal(err) } }
func (qt *QuadTreeDef) SaveToNoms(vw types.ValueWriter, start time.Time, quiet bool) *SQuadTree { wChan := make(chan *SQuadTree, 1024) var wg sync.WaitGroup for i := 0; i < 32; i++ { wg.Add(1) go func() { for sqt := range wChan { vw.WriteValue(*sqt) } wg.Done() }() } if util.MaybeStartCPUProfile() { defer util.StopCPUProfile() } sqt := qt.saveNodeToNoms(wChan, vw, start, quiet) close(wChan) wg.Wait() return sqt }
func main() { flag.Usage = picasaUsage flag.Parse() cachingHTTPClient = util.CachingHttpClient() if *tokenFlag == "" || cachingHTTPClient == nil { flag.Usage() os.Exit(1) } util.MaybeStartCPUProfile() defer util.StopCPUProfile() ds = clientFlags.CreateDataset() if ds == nil { flag.Usage() os.Exit(1) } defer ds.Store().Close() if err := clientFlags.CreateProgressFile(); err != nil { fmt.Println(err) } else { defer clientFlags.CloseProgressFile() } token := oauth2.Token{AccessToken: *tokenFlag} authHTTPClient = oauth2.NewClient(oauth2.NoContext, oauth2.StaticTokenSource(&token)) start = time.Now() user := getUser() userRef := ds.Store().WriteValue(user) fmt.Printf("userRef: %s\n", userRef) _, err := ds.Commit(userRef) d.Exp.NoError(err) }
func main() { err := d.Try(func() { dsFlags := dataset.NewFlags() flag.Usage = customUsage flag.Parse() ds := dsFlags.CreateDataset() dir := flag.Arg(0) if ds == nil || dir == "" { flag.Usage() return } defer ds.Store().Close() if util.MaybeStartCPUProfile() { defer util.StopCPUProfile() } cpuCount := runtime.NumCPU() runtime.GOMAXPROCS(cpuCount) filesChan := make(chan fileIndex, 1024) refsChan := make(chan refIndex, 1024) getFilePaths := func() { index := 0 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { d.Exp.NoError(err, "Cannot traverse directories") if !info.IsDir() && filepath.Ext(path) == ".xml" { filesChan <- fileIndex{path, index} index++ } return nil }) d.Exp.NoError(err) close(filesChan) } wg := sync.WaitGroup{} importXml := func() { expectedType := util.NewMapOfStringToValue() for f := range filesChan { file, err := os.Open(f.path) d.Exp.NoError(err, "Error getting XML") xmlObject, err := mxj.NewMapXmlReader(file) d.Exp.NoError(err, "Error decoding XML") object := xmlObject.Old() file.Close() nomsObj := util.NomsValueFromDecodedJSON(object) d.Chk.IsType(expectedType, nomsObj) r := ref.Ref{} if !*noIO { r = ds.Store().WriteValue(nomsObj).TargetRef() } refsChan <- refIndex{r, f.index} } wg.Done() } go getFilePaths() for i := 0; i < cpuCount*8; i++ { wg.Add(1) go importXml() } go func() { wg.Wait() close(refsChan) // done converting xml to noms }() refList := refIndexList{} for r := range refsChan { refList = append(refList, r) } sort.Sort(refList) refs := make(util.ListOfRefOfMapOfStringToValueDef, len(refList)) for idx, r := range refList { refs[idx] = r.ref } if !*noIO { _, err := ds.Commit(refs.New()) d.Exp.NoError(err) } util.MaybeWriteMemProfile() }) if err != nil { log.Fatal(err) } }
func main() { runtime.GOMAXPROCS(runtime.NumCPU()) flag.Usage = func() { fmt.Printf("Usage: %s -ldb=/path/to/db -ds=sfcrime -input-file=/path/to/data.csv\n\n", os.Args[0]) flag.PrintDefaults() } dsFlags := dataset.NewFlags() flag.Parse() ds := dsFlags.CreateDataset() if ds == nil || *inputFlag == "" { flag.Usage() return } defer ds.Store().Close() csvfile, err := os.Open(*inputFlag) if err != nil { fmt.Println(err) return } defer csvfile.Close() if util.MaybeStartCPUProfile() { defer util.StopCPUProfile() } reader := csv.NewReader(csvfile) minLon := float32(180) minLat := float32(90) maxLat := float32(-90) maxLon := float32(-180) // read the header row and discard it _, err = reader.Read() outLiers := 0 limitExceeded := false iChan, rChan := getNomsWriter(ds.Store()) refList := refIndexList{} refs := []types.Value{} // Start a go routine to add incident refs to the list as they are ready var refWg sync.WaitGroup refWg.Add(1) go func() { for ref := range rChan { refList = append(refList, ref) } sort.Sort(refList) for _, r := range refList { refs = append(refs, r.ref) } refWg.Done() }() index := 0 for r, err := reader.Read(); !limitExceeded && err == nil; r, err = reader.Read() { rowsRead++ id, _ := strconv.ParseInt(r[0], 10, 64) lon64, _ := strconv.ParseFloat(r[9], 32) lat64, _ := strconv.ParseFloat(r[10], 32) geopos := common.GeopositionDef{Latitude: float32(lat64), Longitude: float32(lon64)} incident := common.IncidentDef{ ID: id, Category: r[1], Description: r[2], DayOfWeek: r[3], Date: r[4], Time: r[5], PdDistrict: r[6], Resolution: r[7], Address: r[8], Geoposition: geopos, PdID: r[12], } if geopos.Latitude > 35 && geopos.Latitude < 40 && geopos.Longitude > -125 && geopos.Longitude < 120 { minLat = min(minLat, geopos.Latitude) maxLat = max(maxLat, geopos.Latitude) minLon = min(minLon, geopos.Longitude) maxLon = max(maxLon, geopos.Longitude) iChan <- incidentWithIndex{&incident, index} index++ } else { outLiers++ } if !*quietFlag && rowsRead%maxListSize == 0 { fmt.Printf("Processed %d rows, %d incidents, elapsed time: %.2f secs\n", rowsRead, numIncidents, time.Now().Sub(start).Seconds()) } if rowsRead >= *limitFlag { limitExceeded = true } } close(iChan) refWg.Wait() incidentRefs := types.NewList(refs...) if !*quietFlag { fmt.Printf("Converting refs list to noms list: %.2f secs\n", time.Now().Sub(start).Seconds()) } _, err = ds.Commit(ds.Store().WriteValue(incidentRefs)) d.Exp.NoError(err) if !*quietFlag { fmt.Printf("Commit completed, elaspsed time: %.2f secs\n", time.Now().Sub(start).Seconds()) printDataStats(rowsRead, numIncidents, outLiers, minLat, minLon, maxLat, maxLon) } fmt.Printf("Ref of list containing Incidents: %s, , elaspsed time: %.2f secs\n", incidentRefs.Ref(), time.Now().Sub(start).Seconds()) }