Example #1
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	flags := datas.NewFlags()
	flag.Parse()

	ds, ok := flags.CreateDataStore()
	if !ok || *inputID == "" || *outputID == "" {
		flag.Usage()
		return
	}
	defer ds.Close()

	err := d.Try(func() {
		if util.MaybeStartCPUProfile() {
			defer util.StopCPUProfile()
		}
		inputDataset := dataset.NewDataset(ds, *inputID)
		outputDataset := dataset.NewDataset(ds, *outputID)

		input := inputDataset.Head().Value().(ListOfRefOfMapOfStringToValue)
		output := getIndex(input, ds)

		_, err := outputDataset.Commit(output)
		d.Exp.NoError(err)

		util.MaybeWriteMemProfile()
	})
	if err != nil {
		log.Fatal(err)
	}
}
Example #2
0
func (s *testSuite) TestShove() {
	s.LdbFlagName = "-source-ldb"
	sn := "storeName"
	source1 := dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(s.LdbDir, sn, 1, false)), "foo")
	source1, err := source1.Commit(types.Int32(42))
	s.NoError(err)
	source2, err := source1.Commit(types.Int32(43))
	s.NoError(err)
	source1HeadRef := source1.Head().Ref()
	source2.Store().Close() // Close DataStore backing both Datasets

	ldb2dir := path.Join(s.TempDir, "ldb2")
	out := s.Run(main, []string{"-source-store", sn, "-source", source1HeadRef.String(), "-sink-ldb", ldb2dir, "-sink-store", sn, "-sink-ds", "bar"})
	s.Equal("", out)

	dest := dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(ldb2dir, sn, 1, false)), "bar")
	s.True(types.Int32(42).Equals(dest.Head().Value()))
	dest.Store().Close()

	out = s.Run(main, []string{"-source-store", sn, "-source", "foo", "-sink-ldb", ldb2dir, "-sink-ds", "bar"})
	s.Equal("", out)

	dest = dataset.NewDataset(datas.NewDataStore(chunks.NewLevelDBStore(ldb2dir, sn, 1, false)), "bar")
	s.True(types.Int32(43).Equals(dest.Head().Value()))
	dest.Store().Close()
}
Example #3
0
func (s *testSuite) TestCSVImporterWithPipe() {
	oldDelimiter := delimiter
	newDelimiter := "|"
	delimiter = &newDelimiter
	defer func() { delimiter = oldDelimiter }()

	input, err := ioutil.TempFile(s.TempDir, "")
	d.Chk.NoError(err)
	defer input.Close()
	defer os.Remove(input.Name())

	_, err = input.WriteString("a|b\n1|2\n")
	d.Chk.NoError(err)

	storeName := "store"
	setName := "csv"
	out := s.Run(main, []string{"-store", storeName, "-column-types", "String,Uint8", "-ds", setName, input.Name()})
	s.Equal("", out)

	cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
	ds := dataset.NewDataset(datas.NewDataStore(cs), setName)
	defer ds.Store().Close()
	defer os.RemoveAll(s.LdbDir)

	l := ds.Head().Value().(types.List)
	s.Equal(uint64(1), l.Len())
	v := l.Get(0)
	st := v.(types.Struct)
	s.Equal(types.NewString("1"), st.Get("a"))
	s.Equal(types.Uint8(2), st.Get("b"))
}
Example #4
0
func TestCommitNewPackages(t *testing.T) {
	assert := assert.New(t)
	ds := datas.NewDataStore(chunks.NewMemoryStore())
	pkgDS := dataset.NewDataset(ds, "packages")

	dir, err := ioutil.TempDir("", "")
	assert.NoError(err)
	defer os.RemoveAll(dir)
	inFile := filepath.Join(dir, "in.noms")
	err = ioutil.WriteFile(inFile, []byte("struct Simple{a:Bool}"), 0600)
	assert.NoError(err)

	p := parsePackageFile("name", inFile, pkgDS)
	localPkgs := refSet{p.Ref(): true}
	pkgDS = generate("name", inFile, filepath.Join(dir, "out.go"), dir, map[string]bool{}, p, localPkgs, pkgDS)
	s := pkgDS.Head().Value().(types.SetOfRefOfPackage)
	assert.EqualValues(1, s.Len())
	tr := s.First().TargetValue(ds).Types()[0]
	assert.EqualValues(types.StructKind, tr.Kind())
}
Example #5
0
func (s *testSuite) TestCSVImporter() {
	input, err := ioutil.TempFile(s.TempDir, "")
	d.Chk.NoError(err)
	defer input.Close()
	defer os.Remove(input.Name())

	_, err = input.WriteString("a,b\n")
	d.Chk.NoError(err)
	for i := 0; i < 100; i++ {
		_, err = input.WriteString(fmt.Sprintf("a%d,%d\n", i, i))
		d.Chk.NoError(err)
	}
	_, err = input.Seek(0, 0)
	d.Chk.NoError(err)

	storeName := "store"
	setName := "csv"
	out := s.Run(main, []string{"-store", storeName, "-column-types", "String,Uint8", "-ds", setName, input.Name()})
	s.Equal("", out)

	cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
	ds := dataset.NewDataset(datas.NewDataStore(cs), setName)
	defer ds.Store().Close()
	defer os.RemoveAll(s.LdbDir)

	l := ds.Head().Value().(types.List)
	s.Equal(uint64(100), l.Len())

	i := uint64(0)
	l.IterAll(func(v types.Value, j uint64) {
		s.Equal(i, j)
		st := v.(types.Struct)
		s.Equal(types.NewString(fmt.Sprintf("a%d", i)), st.Get("a"))
		s.Equal(types.Uint8(i), st.Get("b"))
		i++
	})
}
Example #6
0
func TestGetAlbums(t *testing.T) {
	assert := assert.New(t)
	store := datas.NewDataStore(chunks.NewMemoryStore())
	testDs := dataset.NewDataset(store, "test")
	ds = &testDs
	progress := progressTracker{}
	methods := map[string]string{
		"flickr.photosets.getList": `{
			"photosets": {
				"photoset": [
					{
						"id": "42",
						"photos": 2,
						"title": {
							"_content": "My Photoset"
						},
						"description": {
							"_content": ""
						}
					}
				]
			}
		}`,
		"flickr.photosets.getInfo": `{
			"photoset": {
				"id": "42",
				"username": "me",
				"photos": 2,
				"title": {
					"_content": "My Photoset"
				},
				"description": {
					"_content": ""
				}
			}
		}`,
		"flickr.photosets.getPhotos": `{
			"photoset": {
				"id": "42",
				"photo": [
					{
						"id": "0",
						"title": "_0",
						"datetaken": "2011-08-13 04:54:40",
						"url_s": "https:\/\/staticflickr.com\/0\/0.jpg",
						"height_s": "159",
						"width_s": "240",
						"url_m": "https:\/\/staticflickr.com\/0\/1.jpg",
						"height_m": "332",
						"width_m": "500",
						"url_l": "https:\/\/staticflickr.com\/0\/2.jpg",
						"height_l": "679",
						"width_l": "1024",
						"url_o": "https:\/\/staticflickr.com\/0\/3.jpg",
						"height_o": "679",
						"width_o": "1024",
						"longitude": 0,
						"latitude": 0
					},
					{
						"id": "1",
						"title": "_1",
						"datetaken": "2011-12-13 04:51:08",
						"url_s": "https:\/\/staticflickr.com\/1\/0.jpg",
						"height_s": "159",
						"width_s": "240",
						"url_m": "https:\/\/staticflickr.com\/1\/1.jpg",
						"height_m": "332",
						"width_m": "500",
						"url_l": "https:\/\/staticflickr.com\/1\/2.jpg",
						"height_l": "679",
						"width_l": "1024",
						"url_o": "https:\/\/staticflickr.com\/1\/3.jpg",
						"height_o": "6790",
						"width_o": "10240",
						"latitude": 48.8582641,
						"longitude": 2.2923184
					}
				],
				"title": "My Photoset"
			}
		}`,
	}

	albums := getAlbums(fakeFlickrAPI{methods}, &progress)
	assert.Equal(uint64(1), albums.Len())

	album := albums.Get("42").TargetValue(store)
	assert.Equal("42", album.Id())
	assert.Equal("My Photoset", album.Title())

	photos := album.Photos()
	assert.Equal(uint64(2), photos.Len())

	var photo0, photo1 RemotePhoto
	photos.IterAll(func(photo RefOfRemotePhoto) {
		p := photo.TargetValue(store)
		switch id := p.Id(); id {
		case "0":
			photo0 = p
		case "1":
			photo1 = p
		default:
			panic("unexpected photo " + id)
		}
	})

	assert.Equal("0", photo0.Id())
	assert.Equal("_0", photo0.Title())
	assert.Equal(int64(1313236480000), photo0.Date().MsSinceEpoch())
	assert.Equal(float32(0), photo0.Geoposition().Latitude())
	assert.Equal(float32(0), photo0.Geoposition().Longitude())
	assert.Equal(uint64(3), photo0.Sizes().Len()) // two of the images are the same
	assert.Equal(uint64(0), photo0.Tags().Len())

	assert.Equal("1", photo1.Id())
	assert.Equal("_1", photo1.Title())
	// This photo was taken in Paris (by finding the lat/long of the Eiffel Tower), so its date should be interpreted according to that timezone, which is 9 hours ahead of PST (as of this moment).
	assert.Equal(int64(1323780668000-(9000*3600)), photo1.Date().MsSinceEpoch())
	assert.Equal(float32(48.8582641), photo1.Geoposition().Latitude())
	assert.Equal(float32(2.2923184), photo1.Geoposition().Longitude())
	assert.Equal(uint64(4), photo1.Sizes().Len()) // all images are different sizes
	assert.Equal(uint64(0), photo1.Tags().Len())
}
Example #7
0
// FIXME: run with pipe
func (s *testSuite) TestCSVExporter() {
	storeName := "store"
	setName := "csv"
	header := []string{"a", "b", "c"}
	payload := [][]string{
		[]string{"5", "7", "100"},
		[]string{"4", "10", "255"},
		[]string{"512", "12", "55"},
	}
	structName := "SomeStruct"

	// Setup data store
	cs := chunks.NewLevelDBStore(s.LdbDir, storeName, 1, false)
	ds := dataset.NewDataset(datas.NewDataStore(cs), setName)

	// Build Struct fields based on header
	f := make([]types.Field, 0, len(header))
	for _, key := range header {
		f = append(f, types.Field{
			Name: key,
			T:    types.MakePrimitiveType(types.StringKind),
		})
	}

	typeDef := types.MakeStructType(structName, f, types.Choices{})
	pkg := types.NewPackage([]types.Type{typeDef}, []ref.Ref{})
	pkgRef := types.RegisterPackage(&pkg)
	typeRef := types.MakeType(pkgRef, 0)
	structFields := typeDef.Desc.(types.StructDesc).Fields

	// Build data rows
	structs := make([]types.Value, len(payload))
	for i, row := range payload {
		fields := make(map[string]types.Value)
		for j, v := range row {
			fields[structFields[j].Name] = types.NewString(v)
		}
		structs[i] = types.NewStruct(typeRef, typeDef, fields)
	}

	listType := types.MakeCompoundType(types.ListKind, typeRef)
	ds.Commit(types.NewTypedList(listType, structs...))
	ds.Store().Close()

	// Run exporter
	out := s.Run(main, []string{"-store", storeName, "-ds", setName})

	// Verify output
	csvReader := csv.NewReader(strings.NewReader(out))

	row, err := csvReader.Read()
	d.Chk.NoError(err)
	s.Equal(header, row)

	for i := 0; i < len(payload); i++ {
		row, err := csvReader.Read()
		d.Chk.NoError(err)
		s.Equal(payload[i], row)
	}

	row, err = csvReader.Read()
	s.Equal(io.EOF, err)
}
Example #8
0
func main() {
	flags := datas.NewFlags()
	flag.Parse()

	ds, ok := flags.CreateDataStore()
	if !ok {
		ds = datas.NewDataStore(chunks.NewMemoryStore())
	}
	defer ds.Close()

	if *pkgDSFlag != "" {
		if !ok {
			log.Print("Package dataset provided, but DataStore could not be opened.")
			flag.Usage()
			return
		}
	} else {
		log.Print("No package dataset provided; will be unable to process imports.")
		*pkgDSFlag = "default"
	}

	if *outputLang != goExt && *outputLang != jsExt {
		log.Print("Invalid out-lang provided.")
		flag.Usage()
		return
	}

	pkgDS := dataset.NewDataset(ds, *pkgDSFlag)
	// Ensure that, if pkgDS has stuff in it, its head is a SetOfRefOfPackage.
	if h, ok := pkgDS.MaybeHead(); ok {
		d.Chk.IsType(types.SetOfRefOfPackage{}, h.Value())
	}

	localPkgs := refSet{}
	outDir, err := filepath.Abs(*outDirFlag)
	d.Chk.NoError(err, "Could not canonicalize -out-dir: %v", err)
	packageName := ""
	if *outputLang == goExt {
		packageName = getGoPackageName(outDir)
	}

	if *inFlag != "" {
		out := getOutFileName(filepath.Base(*inFlag))
		p := parsePackageFile(packageName, *inFlag, pkgDS)
		localPkgs[p.Ref()] = true
		generate(packageName, *inFlag, filepath.Join(outDir, out), outDir, map[string]bool{}, p, localPkgs, pkgDS)
		return
	}

	// Generate code from all .noms file in the current directory
	nomsFiles, err := filepath.Glob("*" + ext)
	d.Chk.NoError(err)

	written := map[string]bool{}
	packages := map[string]pkg.Parsed{}
	for _, inFile := range nomsFiles {
		p := parsePackageFile(packageName, inFile, pkgDS)
		localPkgs[p.Ref()] = true
		packages[inFile] = p
	}
	// Sort to have deterministic output.
	keys := make([]string, 0, len(packages))
	sort.Strings(keys)
	for inFile := range packages {
		keys = append(keys, inFile)
	}
	for _, inFile := range keys {
		p := packages[inFile]
		pkgDS = generate(packageName, inFile, filepath.Join(outDir, getOutFileName(inFile)), outDir, written, p, localPkgs, pkgDS)
	}
}
Example #9
0
File: main.go Project: arv/noms-old
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Usage = func() {
		fmt.Printf("Usage: %s -ldb=/path/to/db -input-ref=sha1-xyz -output-ds=quadtree\n\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	start := time.Now()

	datastore, ok := datasFlags.CreateDataStore()
	if !ok || *inputRefStr == "" || *outputDs == "" {
		flag.Usage()
		return
	}
	defer datastore.Close()
	dataset := dataset.NewDataset(datastore, *outputDs)

	var inputRef ref.Ref
	err := d.Try(func() {
		inputRef = ref.Parse(*inputRefStr)
	})
	if err != nil {
		log.Fatalf("Invalid inputRef: %v", *inputRefStr)
	}

	gr := common.GeorectangleDef{
		TopLeft:     common.GeopositionDef{Latitude: 37.83, Longitude: -122.52},
		BottomRight: common.GeopositionDef{Latitude: 37.70, Longitude: -122.36},
	}
	qtRoot := common.QuadTreeDef{
		Nodes:          common.ListOfNodeDef{},
		Tiles:          common.MapOfStringToQuadTreeDef{},
		Depth:          0,
		NumDescendents: 0,
		Path:           "",
		Georectangle:   gr,
	}
	if !*quietFlag {
		fmt.Printf("quadTreeRoot: %+v\n", qtRoot.Georectangle)
	}

	nChan := make(chan *common.NodeDef, 1024)
	nodesConverted := uint32(0)

	type hasGeoposition interface {
		Geoposition() common.Geoposition
	}

	go func() {
		ds := dataset.Store()
		walk.SomeP(ds.ReadValue(inputRef), ds, func(v types.Value) (stop bool) {
			var g common.Geoposition

			switch v := v.(type) {
			case hasGeoposition:
				g = v.Geoposition()
			case types.Struct:
				if mg, ok := v.MaybeGet("geo"); ok {
					if mg, ok := mg.(common.Geoposition); ok {
						g = mg
						break
					}
				}
			default:
				return
			}

			// TODO: This check is mega bummer. We really only want to consider RefOfStruct, but it's complicated to filter the case of an inline struct out.
			if !ds.Has(v.Ref()) {
				return
			}

			stop = true

			nodeDef := &common.NodeDef{Geoposition: g.Def(), Reference: v.Ref()}
			nChan <- nodeDef
			nConverted := atomic.AddUint32(&nodesConverted, 1)
			if !*quietFlag && nConverted%1e5 == 0 {
				fmt.Printf("Nodes Converted: %d, elapsed time: %.2f secs\n", nodesConverted, SecsSince(start))
			}
			return
		}, 64)
		close(nChan)
	}()

	nodesAppended := uint32(0)
	for nodeDef := range nChan {
		qtRoot.Append(nodeDef)
		nodesAppended++
		if !*quietFlag && nodesAppended%1e5 == 0 {
			fmt.Printf("Nodes Appended: %d, elapsed time: %.2f secs\n", nodesAppended, SecsSince(start))
			qtRoot.Analyze()
		}
	}

	if !*quietFlag {
		fmt.Printf("Nodes Appended: %d, elapsed time: %.2f secs\n", nodesAppended, SecsSince(start))
		qtRoot.Analyze()
		fmt.Printf("Calling SaveToNoms(), elapsed time: %.2f secs\n", SecsSince(start))
	}

	nomsQtRoot := qtRoot.SaveToNoms(dataset.Store(), start, *quietFlag)
	if !*quietFlag {
		fmt.Printf("Calling Commit(), elapsed time: %.2f secs\n", SecsSince(start))
	}
	_, err = dataset.Commit(types.NewRef(nomsQtRoot.Ref()))
	d.Exp.NoError(err)
	if !*quietFlag {
		fmt.Printf("Commit completed, elapsed time: %.2f secs\n", time.Now().Sub(start).Seconds())
	}
	fmt.Println("QuadTree ref:", nomsQtRoot.Ref())
}