Exemple #1
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	flags := datas.NewFlags()
	flag.Parse()

	ds, ok := flags.CreateDataStore()
	if !ok || *inputID == "" || *outputID == "" {
		flag.Usage()
		return
	}
	defer ds.Close()

	err := d.Try(func() {
		if util.MaybeStartCPUProfile() {
			defer util.StopCPUProfile()
		}
		inputDataset := dataset.NewDataset(ds, *inputID)
		outputDataset := dataset.NewDataset(ds, *outputID)

		input := inputDataset.Head().Value().(ListOfRefOfMapOfStringToValue)
		output := getIndex(input, ds)

		_, err := outputDataset.Commit(output)
		d.Exp.NoError(err)

		util.MaybeWriteMemProfile()
	})
	if err != nil {
		log.Fatal(err)
	}
}
Exemple #2
0
func main() {
	cpuCount := runtime.NumCPU()
	runtime.GOMAXPROCS(cpuCount)

	flag.Parse()

	sourceStore, ok := sourceStoreFlags.CreateDataStore()
	sink := sinkDsFlags.CreateDataset()
	if !ok || sink == nil || *p == 0 || *sourceObject == "" {
		flag.Usage()
		return
	}
	defer sourceStore.Close()
	defer sink.Store().Close()

	err := d.Try(func() {
		if util.MaybeStartCPUProfile() {
			defer util.StopCPUProfile()
		}

		sourceRef := ref.Ref{}
		if r, ok := ref.MaybeParse(*sourceObject); ok {
			if sourceStore.Has(r) {
				sourceRef = r
			}
		} else {
			if c, ok := sourceStore.MaybeHead(*sourceObject); ok {
				sourceRef = c.Ref()
			}
		}
		d.Exp.False(sourceRef.IsEmpty(), "Unknown source object: %s", *sourceObject)

		var err error
		*sink, err = sink.Pull(sourceStore, sourceRef, int(*p))

		util.MaybeWriteMemProfile()
		d.Exp.NoError(err)
	})

	if err != nil {
		log.Fatal(err)
	}
}
Exemple #3
0
func (qt *QuadTreeDef) SaveToNoms(vw types.ValueWriter, start time.Time, quiet bool) *SQuadTree {
	wChan := make(chan *SQuadTree, 1024)
	var wg sync.WaitGroup
	for i := 0; i < 32; i++ {
		wg.Add(1)
		go func() {
			for sqt := range wChan {
				vw.WriteValue(*sqt)
			}
			wg.Done()
		}()
	}

	if util.MaybeStartCPUProfile() {
		defer util.StopCPUProfile()
	}

	sqt := qt.saveNodeToNoms(wChan, vw, start, quiet)
	close(wChan)
	wg.Wait()
	return sqt
}
Exemple #4
0
func main() {
	flag.Usage = picasaUsage
	flag.Parse()
	cachingHTTPClient = util.CachingHttpClient()

	if *tokenFlag == "" || cachingHTTPClient == nil {
		flag.Usage()
		os.Exit(1)
	}

	util.MaybeStartCPUProfile()
	defer util.StopCPUProfile()

	ds = clientFlags.CreateDataset()
	if ds == nil {
		flag.Usage()
		os.Exit(1)
	}
	defer ds.Store().Close()

	if err := clientFlags.CreateProgressFile(); err != nil {
		fmt.Println(err)
	} else {
		defer clientFlags.CloseProgressFile()
	}

	token := oauth2.Token{AccessToken: *tokenFlag}
	authHTTPClient = oauth2.NewClient(oauth2.NoContext, oauth2.StaticTokenSource(&token))

	start = time.Now()
	user := getUser()

	userRef := ds.Store().WriteValue(user)
	fmt.Printf("userRef: %s\n", userRef)
	_, err := ds.Commit(userRef)
	d.Exp.NoError(err)
}
Exemple #5
0
func main() {
	err := d.Try(func() {
		dsFlags := dataset.NewFlags()
		flag.Usage = customUsage
		flag.Parse()
		ds := dsFlags.CreateDataset()
		dir := flag.Arg(0)
		if ds == nil || dir == "" {
			flag.Usage()
			return
		}
		defer ds.Store().Close()

		if util.MaybeStartCPUProfile() {
			defer util.StopCPUProfile()
		}

		cpuCount := runtime.NumCPU()
		runtime.GOMAXPROCS(cpuCount)

		filesChan := make(chan fileIndex, 1024)
		refsChan := make(chan refIndex, 1024)

		getFilePaths := func() {
			index := 0
			err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
				d.Exp.NoError(err, "Cannot traverse directories")
				if !info.IsDir() && filepath.Ext(path) == ".xml" {
					filesChan <- fileIndex{path, index}
					index++
				}

				return nil
			})
			d.Exp.NoError(err)
			close(filesChan)
		}

		wg := sync.WaitGroup{}
		importXml := func() {
			expectedType := util.NewMapOfStringToValue()
			for f := range filesChan {
				file, err := os.Open(f.path)
				d.Exp.NoError(err, "Error getting XML")

				xmlObject, err := mxj.NewMapXmlReader(file)
				d.Exp.NoError(err, "Error decoding XML")
				object := xmlObject.Old()
				file.Close()

				nomsObj := util.NomsValueFromDecodedJSON(object)
				d.Chk.IsType(expectedType, nomsObj)
				r := ref.Ref{}

				if !*noIO {
					r = ds.Store().WriteValue(nomsObj).TargetRef()
				}

				refsChan <- refIndex{r, f.index}
			}

			wg.Done()
		}

		go getFilePaths()
		for i := 0; i < cpuCount*8; i++ {
			wg.Add(1)
			go importXml()
		}
		go func() {
			wg.Wait()
			close(refsChan) // done converting xml to noms
		}()

		refList := refIndexList{}
		for r := range refsChan {
			refList = append(refList, r)
		}
		sort.Sort(refList)

		refs := make(util.ListOfRefOfMapOfStringToValueDef, len(refList))
		for idx, r := range refList {
			refs[idx] = r.ref
		}

		if !*noIO {
			_, err := ds.Commit(refs.New())
			d.Exp.NoError(err)
		}

		util.MaybeWriteMemProfile()
	})

	if err != nil {
		log.Fatal(err)
	}
}
Exemple #6
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Usage = func() {
		fmt.Printf("Usage: %s -ldb=/path/to/db -ds=sfcrime -input-file=/path/to/data.csv\n\n", os.Args[0])
		flag.PrintDefaults()
	}

	dsFlags := dataset.NewFlags()
	flag.Parse()
	ds := dsFlags.CreateDataset()
	if ds == nil || *inputFlag == "" {
		flag.Usage()
		return
	}
	defer ds.Store().Close()

	csvfile, err := os.Open(*inputFlag)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer csvfile.Close()

	if util.MaybeStartCPUProfile() {
		defer util.StopCPUProfile()
	}

	reader := csv.NewReader(csvfile)

	minLon := float32(180)
	minLat := float32(90)
	maxLat := float32(-90)
	maxLon := float32(-180)

	// read the header row and discard it
	_, err = reader.Read()
	outLiers := 0
	limitExceeded := false
	iChan, rChan := getNomsWriter(ds.Store())
	refList := refIndexList{}
	refs := []types.Value{}

	// Start a go routine to add incident refs to the list as they are ready
	var refWg sync.WaitGroup
	refWg.Add(1)
	go func() {
		for ref := range rChan {
			refList = append(refList, ref)
		}
		sort.Sort(refList)
		for _, r := range refList {
			refs = append(refs, r.ref)
		}
		refWg.Done()
	}()

	index := 0

	for r, err := reader.Read(); !limitExceeded && err == nil; r, err = reader.Read() {
		rowsRead++
		id, _ := strconv.ParseInt(r[0], 10, 64)
		lon64, _ := strconv.ParseFloat(r[9], 32)
		lat64, _ := strconv.ParseFloat(r[10], 32)
		geopos := common.GeopositionDef{Latitude: float32(lat64), Longitude: float32(lon64)}
		incident := common.IncidentDef{
			ID:          id,
			Category:    r[1],
			Description: r[2],
			DayOfWeek:   r[3],
			Date:        r[4],
			Time:        r[5],
			PdDistrict:  r[6],
			Resolution:  r[7],
			Address:     r[8],
			Geoposition: geopos,
			PdID:        r[12],
		}

		if geopos.Latitude > 35 && geopos.Latitude < 40 && geopos.Longitude > -125 && geopos.Longitude < 120 {
			minLat = min(minLat, geopos.Latitude)
			maxLat = max(maxLat, geopos.Latitude)
			minLon = min(minLon, geopos.Longitude)
			maxLon = max(maxLon, geopos.Longitude)
			iChan <- incidentWithIndex{&incident, index}
			index++
		} else {
			outLiers++
		}
		if !*quietFlag && rowsRead%maxListSize == 0 {
			fmt.Printf("Processed %d rows, %d incidents, elapsed time: %.2f secs\n", rowsRead, numIncidents, time.Now().Sub(start).Seconds())
		}
		if rowsRead >= *limitFlag {
			limitExceeded = true
		}
	}

	close(iChan)
	refWg.Wait()

	incidentRefs := types.NewList(refs...)
	if !*quietFlag {
		fmt.Printf("Converting refs list to noms list: %.2f secs\n", time.Now().Sub(start).Seconds())
	}

	_, err = ds.Commit(ds.Store().WriteValue(incidentRefs))
	d.Exp.NoError(err)

	if !*quietFlag {
		fmt.Printf("Commit completed, elaspsed time: %.2f secs\n", time.Now().Sub(start).Seconds())
		printDataStats(rowsRead, numIncidents, outLiers, minLat, minLon, maxLat, maxLon)
	}

	fmt.Printf("Ref of list containing Incidents: %s, , elaspsed time: %.2f secs\n", incidentRefs.Ref(), time.Now().Sub(start).Seconds())
}