Esempio n. 1
0
func (suite *LibTestSuite) TestCompositeTypes() {
	// [false true]
	suite.EqualValues(
		types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
		NomsValueFromDecodedJSON([]interface{}{false, true}))

	// [[false true]]
	suite.EqualValues(
		types.NewList().Append(
			types.NewList().Append(types.Bool(false)).Append(types.Bool(true))),
		NomsValueFromDecodedJSON([]interface{}{[]interface{}{false, true}}))

	// {"string": "string",
	//  "list": [false true],
	//  "map": {"nested": "string"}
	// }
	m := MapOfStringToValueDef{
		"string": types.NewString("string"),
		"list":   types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
		"map":    MapOfStringToValueDef{"nested": types.NewString("string")}.New(),
	}.New()
	o := NomsValueFromDecodedJSON(map[string]interface{}{
		"string": "string",
		"list":   []interface{}{false, true},
		"map":    map[string]interface{}{"nested": "string"},
	})

	suite.True(m.Equals(o))
}
Esempio n. 2
0
func (suite *WalkTestSuite) SetupTest() {
	suite.vs = types.NewTestValueStore()
	suite.shouldSeeItem = types.NewString("zzz")
	suite.shouldSee = types.NewList(suite.shouldSeeItem)
	suite.deadValue = types.Uint64(0xDEADBEEF)
	suite.mustSkip = types.NewList(suite.deadValue)
}
Esempio n. 3
0
func (suite *WalkAllTestSuite) TestWalkComposites() {
	suite.walkWorker(suite.storeAndRef(types.NewList()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewList(types.Bool(false), types.Int32(8))), 4)
	suite.walkWorker(suite.storeAndRef(types.NewSet()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewSet(types.Bool(false), types.Int32(8))), 4)
	suite.walkWorker(suite.storeAndRef(types.NewMap()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewMap(types.Int32(8), types.Bool(true), types.Int32(0), types.Bool(false))), 6)
}
Esempio n. 4
0
func (suite *WalkTestSuite) TestStopWalkImmediately() {
	actual := 0
	SomeP(types.NewList(types.NewSet(), types.NewList()), suite.vs, func(v types.Value) bool {
		actual++
		return true
	}, 1)
	suite.Equal(1, actual)
}
Esempio n. 5
0
// Skipping a sub-tree must allow other items in the list to be processed.
func (suite *WalkTestSuite) SkipTestSkipListElement() {
	wholeList := types.NewList(suite.mustSkip, suite.shouldSee, suite.shouldSee)
	reached := suite.skipWorker(wholeList)
	for _, v := range []types.Value{wholeList, suite.mustSkip, suite.shouldSee, suite.shouldSeeItem} {
		suite.Contains(reached, v, "Doesn't contain %+v", v)
	}
	suite.Len(reached, 6)
}
Esempio n. 6
0
func pullDeepRef(t *testing.T, topdown bool) {
	assert := assert.New(t)

	sink := createTestDataset("sink")
	source := createTestDataset("source")

	sourceInitialValue := types.NewList(
		types.NewList(NewList(source)),
		types.NewSet(NewSet(source)),
		types.NewMap(NewMap(source), NewMap(source)))

	source, err := source.Commit(sourceInitialValue)
	assert.NoError(err)

	sink, err = sink.pull(source.Store(), source.Head().Ref(), 1, topdown)
	assert.NoError(err)
	assert.True(source.Head().Equals(sink.Head()))
}
Esempio n. 7
0
func (suite *WalkAllTestSuite) TestWalkNestedComposites() {
	cs := chunks.NewMemoryStore()
	suite.walkWorker(suite.storeAndRef(types.NewList(suite.NewSet(cs), types.Int32(8))), 5)
	suite.walkWorker(suite.storeAndRef(types.NewSet(suite.NewList(cs), suite.NewSet(cs))), 6)
	// {"string": "string",
	//  "list": [false true],
	//  "map": {"nested": "string"}
	//  "mtlist": []
	//  "set": [5 7 8]
	//  []: "wow"
	// }
	nested := types.NewMap(
		types.NewString("string"), types.NewString("string"),
		types.NewString("list"), suite.NewList(cs, types.Bool(false), types.Bool(true)),
		types.NewString("map"), suite.NewMap(cs, types.NewString("nested"), types.NewString("string")),
		types.NewString("mtlist"), suite.NewList(cs),
		types.NewString("set"), suite.NewSet(cs, types.Int32(5), types.Int32(7), types.Int32(8)),
		suite.NewList(cs), types.NewString("wow"), // note that the dupe list chunk is skipped
	)
	suite.walkWorker(suite.storeAndRef(nested), 25)
}
Esempio n. 8
0
func (suite *WalkAllTestSuite) NewList(cs chunks.ChunkStore, vs ...types.Value) types.RefBase {
	v := types.NewList(vs...)
	return suite.vs.WriteValue(v)
}
Esempio n. 9
0
func NewList(ds Dataset, vs ...types.Value) types.RefBase {
	v := types.NewList(vs...)
	return ds.Store().WriteValue(v)
}
Esempio n. 10
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Usage = func() {
		fmt.Printf("Usage: %s -ldb=/path/to/db -ds=sfcrime -input-file=/path/to/data.csv\n\n", os.Args[0])
		flag.PrintDefaults()
	}

	dsFlags := dataset.NewFlags()
	flag.Parse()
	ds := dsFlags.CreateDataset()
	if ds == nil || *inputFlag == "" {
		flag.Usage()
		return
	}
	defer ds.Store().Close()

	csvfile, err := os.Open(*inputFlag)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer csvfile.Close()

	if util.MaybeStartCPUProfile() {
		defer util.StopCPUProfile()
	}

	reader := csv.NewReader(csvfile)

	minLon := float32(180)
	minLat := float32(90)
	maxLat := float32(-90)
	maxLon := float32(-180)

	// read the header row and discard it
	_, err = reader.Read()
	outLiers := 0
	limitExceeded := false
	iChan, rChan := getNomsWriter(ds.Store())
	refList := refIndexList{}
	refs := []types.Value{}

	// Start a go routine to add incident refs to the list as they are ready
	var refWg sync.WaitGroup
	refWg.Add(1)
	go func() {
		for ref := range rChan {
			refList = append(refList, ref)
		}
		sort.Sort(refList)
		for _, r := range refList {
			refs = append(refs, r.ref)
		}
		refWg.Done()
	}()

	index := 0

	for r, err := reader.Read(); !limitExceeded && err == nil; r, err = reader.Read() {
		rowsRead++
		id, _ := strconv.ParseInt(r[0], 10, 64)
		lon64, _ := strconv.ParseFloat(r[9], 32)
		lat64, _ := strconv.ParseFloat(r[10], 32)
		geopos := common.GeopositionDef{Latitude: float32(lat64), Longitude: float32(lon64)}
		incident := common.IncidentDef{
			ID:          id,
			Category:    r[1],
			Description: r[2],
			DayOfWeek:   r[3],
			Date:        r[4],
			Time:        r[5],
			PdDistrict:  r[6],
			Resolution:  r[7],
			Address:     r[8],
			Geoposition: geopos,
			PdID:        r[12],
		}

		if geopos.Latitude > 35 && geopos.Latitude < 40 && geopos.Longitude > -125 && geopos.Longitude < 120 {
			minLat = min(minLat, geopos.Latitude)
			maxLat = max(maxLat, geopos.Latitude)
			minLon = min(minLon, geopos.Longitude)
			maxLon = max(maxLon, geopos.Longitude)
			iChan <- incidentWithIndex{&incident, index}
			index++
		} else {
			outLiers++
		}
		if !*quietFlag && rowsRead%maxListSize == 0 {
			fmt.Printf("Processed %d rows, %d incidents, elapsed time: %.2f secs\n", rowsRead, numIncidents, time.Now().Sub(start).Seconds())
		}
		if rowsRead >= *limitFlag {
			limitExceeded = true
		}
	}

	close(iChan)
	refWg.Wait()

	incidentRefs := types.NewList(refs...)
	if !*quietFlag {
		fmt.Printf("Converting refs list to noms list: %.2f secs\n", time.Now().Sub(start).Seconds())
	}

	_, err = ds.Commit(ds.Store().WriteValue(incidentRefs))
	d.Exp.NoError(err)

	if !*quietFlag {
		fmt.Printf("Commit completed, elaspsed time: %.2f secs\n", time.Now().Sub(start).Seconds())
		printDataStats(rowsRead, numIncidents, outLiers, minLat, minLon, maxLat, maxLon)
	}

	fmt.Printf("Ref of list containing Incidents: %s, , elaspsed time: %.2f secs\n", incidentRefs.Ref(), time.Now().Sub(start).Seconds())
}