func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flags := datas.NewFlags()
flag.Parse()
ds, ok := flags.CreateDataStore()
if !ok || *inputID == "" || *outputID == "" {
flag.Usage()
return
}
defer ds.Close()
err := d.Try(func() {
if util.MaybeStartCPUProfile() {
defer util.StopCPUProfile()
}
inputDataset := dataset.NewDataset(ds, *inputID)
outputDataset := dataset.NewDataset(ds, *outputID)
input := inputDataset.Head().Value().(ListOfRefOfMapOfStringToValue)
output := getIndex(input, ds)
_, err := outputDataset.Commit(output)
d.Exp.NoError(err)
util.MaybeWriteMemProfile()
})
if err != nil {
log.Fatal(err)
}
}
func (s *testSuite) TestShove() {
s.LdbFlagName = "-source-ldb"
sn := "storeName"
source1 := dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(s.LdbDir, sn, 1, false)), "foo")
source1, err := source1.Commit(types.Int32(42))
s.NoError(err)
source2, err := source1.Commit(types.Int32(43))
s.NoError(err)
source1HeadRef := source1.Head().Ref()
source2.Store().Close() // Close DataStore backing both Datasets
ldb2dir := path.Join(s.TempDir, "ldb2")
out := s.Run(main, []string{"-source-store", sn, "-source", source1HeadRef.String(), "-sink-ldb", ldb2dir, "-sink-store", sn, "-sink-ds", "bar"})
s.Equal("", out)
dest := dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(ldb2dir, sn, 1, false)), "bar")
s.True(types.Int32(42).Equals(dest.Head().Value()))
dest.Store().Close()
out = s.Run(main, []string{"-source-store", sn, "-source", "foo", "-sink-ldb", ldb2dir, "-sink-ds", "bar"})
s.Equal("", out)
dest = dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(ldb2dir, sn, 1, false)), "bar")
s.True(types.Int32(43).Equals(dest.Head().Value()))
dest.Store().Close()
}
func (s *testSuite) TestCSVImporterWithPipe() {
oldDelimiter := delimiter
newDelimiter := "|"
delimiter = &newDelimiter
defer func() { delimiter = oldDelimiter }()
input, err := ioutil.TempFile(s.TempDir, "")
d.Chk.NoError(err)
defer input.Close()
defer os.Remove(input.Name())
_, err = input.WriteString("a|b\n1|2\n")
d.Chk.NoError(err)
storeName := "store"
setName := "csv"
out := s.Run(main, []string{"-store", storeName, "-column-types", "String,Uint8", "-ds", setName, input.Name()})
s.Equal("", out)
cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
ds := dataset.NewDataset(datas.NewDataStore(cs), setName)
defer ds.Store().Close()
defer os.RemoveAll(s.LdbDir)
l := ds.Head().Value().(types.List)
s.Equal(uint64(1), l.Len())
v := l.Get(0)
st := v.(types.Struct)
s.Equal(types.NewString("1"), st.Get("a"))
s.Equal(types.Uint8(2), st.Get("b"))
}
func TestCommitNewPackages(t *testing.T) {
assert := assert.New(t)
ds := datas.NewDataStore(chunks.NewMemoryStore())
pkgDS := dataset.NewDataset(ds, "packages")
dir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(dir)
inFile := filepath.Join(dir, "in.noms")
err = ioutil.WriteFile(inFile, []byte("struct Simple{a:Bool}"), 0600)
assert.NoError(err)
p := parsePackageFile("name", inFile, pkgDS)
localPkgs := refSet{p.Ref(): true}
pkgDS = generate("name", inFile, filepath.Join(dir, "out.go"), dir, map[string]bool{}, p, localPkgs, pkgDS)
s := pkgDS.Head().Value().(types.SetOfRefOfPackage)
assert.EqualValues(1, s.Len())
tr := s.First().TargetValue(ds).Types()[0]
assert.EqualValues(types.StructKind, tr.Kind())
}
func (s *testSuite) TestCSVImporter() {
input, err := ioutil.TempFile(s.TempDir, "")
d.Chk.NoError(err)
defer input.Close()
defer os.Remove(input.Name())
_, err = input.WriteString("a,b\n")
d.Chk.NoError(err)
for i := 0; i < 100; i++ {
_, err = input.WriteString(fmt.Sprintf("a%d,%d\n", i, i))
d.Chk.NoError(err)
}
_, err = input.Seek(0, 0)
d.Chk.NoError(err)
storeName := "store"
setName := "csv"
out := s.Run(main, []string{"-store", storeName, "-column-types", "String,Uint8", "-ds", setName, input.Name()})
s.Equal("", out)
cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
ds := dataset.NewDataset(datas.NewDataStore(cs), setName)
defer ds.Store().Close()
defer os.RemoveAll(s.LdbDir)
l := ds.Head().Value().(types.List)
s.Equal(uint64(100), l.Len())
i := uint64(0)
l.IterAll(func(v types.Value, j uint64) {
s.Equal(i, j)
st := v.(types.Struct)
s.Equal(types.NewString(fmt.Sprintf("a%d", i)), st.Get("a"))
s.Equal(types.Uint8(i), st.Get("b"))
i++
})
}
func TestGetAlbums(t *testing.T) {
assert := assert.New(t)
store := datas.NewDataStore(chunks.NewMemoryStore())
testDs := dataset.NewDataset(store, "test")
ds = &testDs
progress := progressTracker{}
methods := map[string]string{
"flickr.photosets.getList": `{
"photosets": {
"photoset": [
{
"id": "42",
"photos": 2,
"title": {
"_content": "My Photoset"
},
"description": {
"_content": ""
}
}
]
}
}`,
"flickr.photosets.getInfo": `{
"photoset": {
"id": "42",
"username": "******",
"photos": 2,
"title": {
"_content": "My Photoset"
},
"description": {
"_content": ""
}
}
}`,
"flickr.photosets.getPhotos": `{
"photoset": {
"id": "42",
"photo": [
{
"id": "0",
"title": "_0",
"datetaken": "2011-08-13 04:54:40",
"url_s": "https:\/\/staticflickr.com\/0\/0.jpg",
"height_s": "159",
"width_s": "240",
"url_m": "https:\/\/staticflickr.com\/0\/1.jpg",
"height_m": "332",
"width_m": "500",
"url_l": "https:\/\/staticflickr.com\/0\/2.jpg",
"height_l": "679",
"width_l": "1024",
"url_o": "https:\/\/staticflickr.com\/0\/3.jpg",
"height_o": "679",
"width_o": "1024",
"longitude": 0,
"latitude": 0
},
{
"id": "1",
"title": "_1",
"datetaken": "2011-12-13 04:51:08",
"url_s": "https:\/\/staticflickr.com\/1\/0.jpg",
"height_s": "159",
"width_s": "240",
"url_m": "https:\/\/staticflickr.com\/1\/1.jpg",
"height_m": "332",
"width_m": "500",
"url_l": "https:\/\/staticflickr.com\/1\/2.jpg",
"height_l": "679",
"width_l": "1024",
"url_o": "https:\/\/staticflickr.com\/1\/3.jpg",
"height_o": "6790",
"width_o": "10240",
"latitude": 48.8582641,
"longitude": 2.2923184
}
],
"title": "My Photoset"
}
}`,
}
albums := getAlbums(fakeFlickrAPI{methods}, &progress)
assert.Equal(uint64(1), albums.Len())
album := albums.Get("42").TargetValue(store)
assert.Equal("42", album.Id())
assert.Equal("My Photoset", album.Title())
photos := album.Photos()
assert.Equal(uint64(2), photos.Len())
var photo0, photo1 RemotePhoto
photos.IterAll(func(photo RefOfRemotePhoto) {
p := photo.TargetValue(store)
switch id := p.Id(); id {
case "0":
photo0 = p
case "1":
photo1 = p
default:
panic("unexpected photo " + id)
}
})
assert.Equal("0", photo0.Id())
assert.Equal("_0", photo0.Title())
assert.Equal(int64(1313236480000), photo0.Date().MsSinceEpoch())
assert.Equal(float32(0), photo0.Geoposition().Latitude())
assert.Equal(float32(0), photo0.Geoposition().Longitude())
assert.Equal(uint64(3), photo0.Sizes().Len()) // two of the images are the same
assert.Equal(uint64(0), photo0.Tags().Len())
assert.Equal("1", photo1.Id())
assert.Equal("_1", photo1.Title())
// This photo was taken in Paris (by finding the lat/long of the Eiffel Tower), so its date should be interpreted according to that timezone, which is 9 hours ahead of PST (as of this moment).
assert.Equal(int64(1323780668000-(9000*3600)), photo1.Date().MsSinceEpoch())
assert.Equal(float32(48.8582641), photo1.Geoposition().Latitude())
assert.Equal(float32(2.2923184), photo1.Geoposition().Longitude())
assert.Equal(uint64(4), photo1.Sizes().Len()) // all images are different sizes
assert.Equal(uint64(0), photo1.Tags().Len())
}
// FIXME: run with pipe
func (s *testSuite) TestCSVExporter() {
storeName := "store"
setName := "csv"
header := []string{"a", "b", "c"}
payload := [][]string{
[]string{"5", "7", "100"},
[]string{"4", "10", "255"},
[]string{"512", "12", "55"},
}
structName := "SomeStruct"
// Setup data store
cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
ds := dataset.NewDataset(datas.NewDataStore(cs), setName)
// Build Struct fields based on header
f := make([]types.Field, 0, len(header))
for _, key := range header {
f = append(f, types.Field{
Name: key,
T: types.MakePrimitiveType(types.StringKind),
})
}
typeDef := types.MakeStructType(structName, f, types.Choices{})
pkg := types.NewPackage([]types.Type{typeDef}, []ref.Ref{})
pkgRef := types.RegisterPackage(&pkg)
typeRef := types.MakeType(pkgRef, 0)
structFields := typeDef.Desc.(types.StructDesc).Fields
// Build data rows
structs := make([]types.Value, len(payload))
for i, row := range payload {
fields := make(map[string]types.Value)
for j, v := range row {
fields[structFields[j].Name] = types.NewString(v)
}
structs[i] = types.NewStruct(typeRef, typeDef, fields)
}
listType := types.MakeCompoundType(types.ListKind, typeRef)
ds.Commit(types.NewTypedList(listType, structs...))
ds.Store().Close()
// Run exporter
out := s.Run(main, []string{"-store", storeName, "-ds", setName})
// Verify output
csvReader := csv.NewReader(strings.NewReader(out))
row, err := csvReader.Read()
d.Chk.NoError(err)
s.Equal(header, row)
for i := 0; i < len(payload); i++ {
row, err := csvReader.Read()
d.Chk.NoError(err)
s.Equal(payload[i], row)
}
row, err = csvReader.Read()
s.Equal(io.EOF, err)
}
func main() {
flags := datas.NewFlags()
flag.Parse()
ds, ok := flags.CreateDataStore()
if !ok {
ds = datas.NewDataStore(chunks.NewMemoryStore())
}
defer ds.Close()
if *pkgDSFlag != "" {
if !ok {
log.Print("Package dataset provided, but DataStore could not be opened.")
flag.Usage()
return
}
} else {
log.Print("No package dataset provided; will be unable to process imports.")
*pkgDSFlag = "default"
}
if *outputLang != goExt && *outputLang != jsExt {
log.Print("Invalid out-lang provided.")
flag.Usage()
return
}
pkgDS := dataset.NewDataset(ds, *pkgDSFlag)
// Ensure that, if pkgDS has stuff in it, its head is a SetOfRefOfPackage.
if h, ok := pkgDS.MaybeHead(); ok {
d.Chk.IsType(types.SetOfRefOfPackage{}, h.Value())
}
localPkgs := refSet{}
outDir, err := filepath.Abs(*outDirFlag)
d.Chk.NoError(err, "Could not canonicalize -out-dir: %v", err)
packageName := ""
if *outputLang == goExt {
packageName = getGoPackageName(outDir)
}
if *inFlag != "" {
out := getOutFileName(filepath.Base(*inFlag))
p := parsePackageFile(packageName, *inFlag, pkgDS)
localPkgs[p.Ref()] = true
generate(packageName, *inFlag, filepath.Join(outDir, out), outDir, map[string]bool{}, p, localPkgs, pkgDS)
return
}
// Generate code from all .noms file in the current directory
nomsFiles, err := filepath.Glob("*" + ext)
d.Chk.NoError(err)
written := map[string]bool{}
packages := map[string]pkg.Parsed{}
for _, inFile := range nomsFiles {
p := parsePackageFile(packageName, inFile, pkgDS)
localPkgs[p.Ref()] = true
packages[inFile] = p
}
// Sort to have deterministic output.
keys := make([]string, 0, len(packages))
sort.Strings(keys)
for inFile := range packages {
keys = append(keys, inFile)
}
for _, inFile := range keys {
p := packages[inFile]
pkgDS = generate(packageName, inFile, filepath.Join(outDir, getOutFileName(inFile)), outDir, written, p, localPkgs, pkgDS)
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Usage = func() {
fmt.Printf("Usage: %s -ldb=/path/to/db -input-ref=sha1-xyz -output-ds=quadtree\n\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
start := time.Now()
datastore, ok := datasFlags.CreateDataStore()
if !ok || *inputRefStr == "" || *outputDs == "" {
flag.Usage()
return
}
defer datastore.Close()
dataset := dataset.NewDataset(datastore, *outputDs)
var inputRef ref.Ref
err := d.Try(func() {
inputRef = ref.Parse(*inputRefStr)
})
if err != nil {
log.Fatalf("Invalid inputRef: %v", *inputRefStr)
}
gr := common.GeorectangleDef{
TopLeft: common.GeopositionDef{Latitude: 37.83, Longitude: -122.52},
BottomRight: common.GeopositionDef{Latitude: 37.70, Longitude: -122.36},
}
qtRoot := common.QuadTreeDef{
Nodes: common.ListOfNodeDef{},
Tiles: common.MapOfStringToQuadTreeDef{},
Depth: 0,
NumDescendents: 0,
Path: "",
Georectangle: gr,
}
if !*quietFlag {
fmt.Printf("quadTreeRoot: %+v\n", qtRoot.Georectangle)
}
nChan := make(chan *common.NodeDef, 1024)
nodesConverted := uint32(0)
type hasGeoposition interface {
Geoposition() common.Geoposition
}
go func() {
ds := dataset.Store()
walk.SomeP(ds.ReadValue(inputRef), ds, func(v types.Value) (stop bool) {
var g common.Geoposition
switch v := v.(type) {
case hasGeoposition:
g = v.Geoposition()
case types.Struct:
if mg, ok := v.MaybeGet("geo"); ok {
if mg, ok := mg.(common.Geoposition); ok {
g = mg
break
}
}
default:
return
}
// TODO: This check is mega bummer. We really only want to consider RefOfStruct, but it's complicated to filter the case of an inline struct out.
if !ds.Has(v.Ref()) {
return
}
stop = true
nodeDef := &common.NodeDef{Geoposition: g.Def(), Reference: v.Ref()}
nChan <- nodeDef
nConverted := atomic.AddUint32(&nodesConverted, 1)
if !*quietFlag && nConverted%1e5 == 0 {
fmt.Printf("Nodes Converted: %d, elapsed time: %.2f secs\n", nodesConverted, SecsSince(start))
}
return
}, 64)
close(nChan)
}()
nodesAppended := uint32(0)
for nodeDef := range nChan {
qtRoot.Append(nodeDef)
nodesAppended++
if !*quietFlag && nodesAppended%1e5 == 0 {
fmt.Printf("Nodes Appended: %d, elapsed time: %.2f secs\n", nodesAppended, SecsSince(start))
qtRoot.Analyze()
}
}
if !*quietFlag {
fmt.Printf("Nodes Appended: %d, elapsed time: %.2f secs\n", nodesAppended, SecsSince(start))
qtRoot.Analyze()
fmt.Printf("Calling SaveToNoms(), elapsed time: %.2f secs\n", SecsSince(start))
}
nomsQtRoot := qtRoot.SaveToNoms(dataset.Store(), start, *quietFlag)
if !*quietFlag {
fmt.Printf("Calling Commit(), elapsed time: %.2f secs\n", SecsSince(start))
}
_, err = dataset.Commit(types.NewRef(nomsQtRoot.Ref()))
d.Exp.NoError(err)
if !*quietFlag {
fmt.Printf("Commit completed, elapsed time: %.2f secs\n", time.Now().Sub(start).Seconds())
}
fmt.Println("QuadTree ref:", nomsQtRoot.Ref())
}