func (suite *LibTestSuite) TestCompositeTypes() {
// [false true]
suite.EqualValues(
types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
NomsValueFromDecodedJSON([]interface{}{false, true}))
// [[false true]]
suite.EqualValues(
types.NewList().Append(
types.NewList().Append(types.Bool(false)).Append(types.Bool(true))),
NomsValueFromDecodedJSON([]interface{}{[]interface{}{false, true}}))
// {"string": "string",
// "list": [false true],
// "map": {"nested": "string"}
// }
m := MapOfStringToValueDef{
"string": types.NewString("string"),
"list": types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
"map": MapOfStringToValueDef{"nested": types.NewString("string")}.New(),
}.New()
o := NomsValueFromDecodedJSON(map[string]interface{}{
"string": "string",
"list": []interface{}{false, true},
"map": map[string]interface{}{"nested": "string"},
})
suite.True(m.Equals(o))
}
func (suite *WalkTestSuite) SetupTest() {
suite.vs = types.NewTestValueStore()
suite.shouldSeeItem = types.NewString("zzz")
suite.shouldSee = types.NewList(suite.shouldSeeItem)
suite.deadValue = types.Uint64(0xDEADBEEF)
suite.mustSkip = types.NewList(suite.deadValue)
}
func (suite *WalkAllTestSuite) TestWalkComposites() {
suite.walkWorker(suite.storeAndRef(types.NewList()), 2)
suite.walkWorker(suite.storeAndRef(types.NewList(types.Bool(false), types.Int32(8))), 4)
suite.walkWorker(suite.storeAndRef(types.NewSet()), 2)
suite.walkWorker(suite.storeAndRef(types.NewSet(types.Bool(false), types.Int32(8))), 4)
suite.walkWorker(suite.storeAndRef(types.NewMap()), 2)
suite.walkWorker(suite.storeAndRef(types.NewMap(types.Int32(8), types.Bool(true), types.Int32(0), types.Bool(false))), 6)
}
func (suite *WalkTestSuite) TestStopWalkImmediately() {
actual := 0
SomeP(types.NewList(types.NewSet(), types.NewList()), suite.vs, func(v types.Value) bool {
actual++
return true
}, 1)
suite.Equal(1, actual)
}
// Skipping a sub-tree must allow other items in the list to be processed.
func (suite *WalkTestSuite) SkipTestSkipListElement() {
wholeList := types.NewList(suite.mustSkip, suite.shouldSee, suite.shouldSee)
reached := suite.skipWorker(wholeList)
for _, v := range []types.Value{wholeList, suite.mustSkip, suite.shouldSee, suite.shouldSeeItem} {
suite.Contains(reached, v, "Doesn't contain %+v", v)
}
suite.Len(reached, 6)
}
func pullDeepRef(t *testing.T, topdown bool) {
assert := assert.New(t)
sink := createTestDataset("sink")
source := createTestDataset("source")
sourceInitialValue := types.NewList(
types.NewList(NewList(source)),
types.NewSet(NewSet(source)),
types.NewMap(NewMap(source), NewMap(source)))
source, err := source.Commit(sourceInitialValue)
assert.NoError(err)
sink, err = sink.pull(source.Store(), source.Head().Ref(), 1, topdown)
assert.NoError(err)
assert.True(source.Head().Equals(sink.Head()))
}
func (suite *WalkAllTestSuite) TestWalkNestedComposites() {
cs := chunks.NewMemoryStore()
suite.walkWorker(suite.storeAndRef(types.NewList(suite.NewSet(cs), types.Int32(8))), 5)
suite.walkWorker(suite.storeAndRef(types.NewSet(suite.NewList(cs), suite.NewSet(cs))), 6)
// {"string": "string",
// "list": [false true],
// "map": {"nested": "string"}
// "mtlist": []
// "set": [5 7 8]
// []: "wow"
// }
nested := types.NewMap(
types.NewString("string"), types.NewString("string"),
types.NewString("list"), suite.NewList(cs, types.Bool(false), types.Bool(true)),
types.NewString("map"), suite.NewMap(cs, types.NewString("nested"), types.NewString("string")),
types.NewString("mtlist"), suite.NewList(cs),
types.NewString("set"), suite.NewSet(cs, types.Int32(5), types.Int32(7), types.Int32(8)),
suite.NewList(cs), types.NewString("wow"), // note that the dupe list chunk is skipped
)
suite.walkWorker(suite.storeAndRef(nested), 25)
}
func (suite *WalkAllTestSuite) NewList(cs chunks.ChunkStore, vs ...types.Value) types.RefBase {
v := types.NewList(vs...)
return suite.vs.WriteValue(v)
}
func NewList(ds Dataset, vs ...types.Value) types.RefBase {
v := types.NewList(vs...)
return ds.Store().WriteValue(v)
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Usage = func() {
fmt.Printf("Usage: %s -ldb=/path/to/db -ds=sfcrime -input-file=/path/to/data.csv\n\n", os.Args[0])
flag.PrintDefaults()
}
dsFlags := dataset.NewFlags()
flag.Parse()
ds := dsFlags.CreateDataset()
if ds == nil || *inputFlag == "" {
flag.Usage()
return
}
defer ds.Store().Close()
csvfile, err := os.Open(*inputFlag)
if err != nil {
fmt.Println(err)
return
}
defer csvfile.Close()
if util.MaybeStartCPUProfile() {
defer util.StopCPUProfile()
}
reader := csv.NewReader(csvfile)
minLon := float32(180)
minLat := float32(90)
maxLat := float32(-90)
maxLon := float32(-180)
// read the header row and discard it
_, err = reader.Read()
outLiers := 0
limitExceeded := false
iChan, rChan := getNomsWriter(ds.Store())
refList := refIndexList{}
refs := []types.Value{}
// Start a go routine to add incident refs to the list as they are ready
var refWg sync.WaitGroup
refWg.Add(1)
go func() {
for ref := range rChan {
refList = append(refList, ref)
}
sort.Sort(refList)
for _, r := range refList {
refs = append(refs, r.ref)
}
refWg.Done()
}()
index := 0
for r, err := reader.Read(); !limitExceeded && err == nil; r, err = reader.Read() {
rowsRead++
id, _ := strconv.ParseInt(r[0], 10, 64)
lon64, _ := strconv.ParseFloat(r[9], 32)
lat64, _ := strconv.ParseFloat(r[10], 32)
geopos := common.GeopositionDef{Latitude: float32(lat64), Longitude: float32(lon64)}
incident := common.IncidentDef{
ID: id,
Category: r[1],
Description: r[2],
DayOfWeek: r[3],
Date: r[4],
Time: r[5],
PdDistrict: r[6],
Resolution: r[7],
Address: r[8],
Geoposition: geopos,
PdID: r[12],
}
if geopos.Latitude > 35 && geopos.Latitude < 40 && geopos.Longitude > -125 && geopos.Longitude < 120 {
minLat = min(minLat, geopos.Latitude)
maxLat = max(maxLat, geopos.Latitude)
minLon = min(minLon, geopos.Longitude)
maxLon = max(maxLon, geopos.Longitude)
iChan <- incidentWithIndex{&incident, index}
index++
} else {
outLiers++
}
if !*quietFlag && rowsRead%maxListSize == 0 {
fmt.Printf("Processed %d rows, %d incidents, elapsed time: %.2f secs\n", rowsRead, numIncidents, time.Now().Sub(start).Seconds())
}
if rowsRead >= *limitFlag {
limitExceeded = true
}
}
close(iChan)
refWg.Wait()
incidentRefs := types.NewList(refs...)
if !*quietFlag {
fmt.Printf("Converting refs list to noms list: %.2f secs\n", time.Now().Sub(start).Seconds())
}
_, err = ds.Commit(ds.Store().WriteValue(incidentRefs))
d.Exp.NoError(err)
if !*quietFlag {
fmt.Printf("Commit completed, elaspsed time: %.2f secs\n", time.Now().Sub(start).Seconds())
printDataStats(rowsRead, numIncidents, outLiers, minLat, minLon, maxLat, maxLon)
}
fmt.Printf("Ref of list containing Incidents: %s, , elaspsed time: %.2f secs\n", incidentRefs.Ref(), time.Now().Sub(start).Seconds())
}