Beispiel #1
0
func TestReadToMap(t *testing.T) {
	assert := assert.New(t)
	ds := datas.NewDatabase(chunks.NewMemoryStore())

	dataString := `a,1,true
b,2,false
`
	r := NewCSVReader(bytes.NewBufferString(dataString), ',')

	headers := []string{"A", "B", "C"}
	kinds := KindSlice{types.StringKind, types.NumberKind, types.BoolKind}
	m := ReadToMap(r, headers, 0, kinds, ds)

	assert.Equal(uint64(2), m.Len())
	assert.True(m.Type().Equals(
		types.MakeMapType(types.StringType, types.MakeStructType("", map[string]*types.Type{
			"B": types.NumberType,
			"C": types.BoolType,
		}))))

	assert.True(m.Get(types.String("a")).Equals(types.NewStruct("", map[string]types.Value{
		"B": types.Number(1),
		"C": types.Bool(true),
	})))
	assert.True(m.Get(types.String("b")).Equals(types.NewStruct("", map[string]types.Value{
		"B": types.Number(2),
		"C": types.Bool(false),
	})))
}
Beispiel #2
0
func TestCommitWithoutMetaField(t *testing.T) {
	assert := assert.New(t)
	metaCommit := types.NewStruct("Commit", types.StructData{
		"value":   types.Number(9),
		"parents": types.NewSet(),
		"meta":    types.EmptyStruct,
	})
	assert.True(IsCommitType(metaCommit.Type()))

	noMetaCommit := types.NewStruct("Commit", types.StructData{
		"value":   types.Number(9),
		"parents": types.NewSet(),
	})
	assert.False(IsCommitType(noMetaCommit.Type()))
}
func TestHandlePostRoot(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	vs := types.NewValueStore(types.NewBatchStoreAdaptor(cs))

	commit := NewCommit(types.String("head"), types.NewSet(), types.NewStruct("Meta", types.StructData{}))
	newHead := types.NewMap(types.String("dataset1"), vs.WriteValue(commit))
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte("abc")),
		types.EncodeValue(newHead, nil),
	}
	err := cs.PutMany(chnx)
	assert.NoError(err)

	// First attempt should fail, as 'last' won't match.
	u := &url.URL{}
	queryParams := url.Values{}
	queryParams.Add("last", chnx[0].Hash().String())
	queryParams.Add("current", chnx[1].Hash().String())
	u.RawQuery = queryParams.Encode()
	url := u.String()

	w := httptest.NewRecorder()
	HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
	assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))

	// Now, update the root manually to 'last' and try again.
	assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
	w = httptest.NewRecorder()
	HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
	assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
Beispiel #4
0
func createStruct(name string, kv ...interface{}) types.Struct {
	fields := types.StructData{}
	for i := 0; i < len(kv); i += 2 {
		fields[kv[i].(string)] = valToTypesValue(kv[i+1])
	}
	return types.NewStruct(name, fields)
}
Beispiel #5
0
func (s *testSuite) TestCSVImporterToMap() {
	input, err := ioutil.TempFile(s.TempDir, "")
	d.Chk.NoError(err)
	defer input.Close()
	defer os.Remove(input.Name())

	_, err = input.WriteString("a,b,c\n")
	d.Chk.NoError(err)
	for i := 0; i < 20; i++ {
		_, err = input.WriteString(fmt.Sprintf("a%d,%d,%d\n", i, i, i*2))
		d.Chk.NoError(err)
	}
	_, err = input.Seek(0, 0)
	d.Chk.NoError(err)

	setName := "csv"
	dataspec := test_util.CreateValueSpecString("ldb", s.LdbDir, setName)
	out := s.Run(main, []string{"-no-progress", "-column-types", "String,Number,Number", "-dest-type", "map:1", dataspec, input.Name()})
	s.Equal("", out)

	cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false)
	ds := dataset.NewDataset(datas.NewDatabase(cs), setName)
	defer ds.Database().Close()
	defer os.RemoveAll(s.LdbDir)

	m := ds.HeadValue().(types.Map)
	s.Equal(uint64(20), m.Len())

	for i := 0; i < 20; i++ {
		m.Get(types.Number(i)).(types.Struct).Equals(types.NewStruct("", map[string]types.Value{
			"a": types.String(fmt.Sprintf("a%d", i)),
			"c": types.Number(i * 2),
		}))
	}
}
Beispiel #6
0
func TestPullWithMeta(t *testing.T) {
	assert := assert.New(t)

	sink := createTestDataset("sink")
	source := createTestDataset("source")

	v1 := types.Number(1)
	m1 := types.NewStruct("Meta", types.StructData{
		"name": types.String("one"),
	})
	source, err := source.Commit(v1, CommitOptions{Meta: m1})
	assert.NoError(err)

	v2 := types.Number(2)
	m2 := types.NewStruct("Meta", types.StructData{
		"name": types.String("two"),
	})
	source, err = source.Commit(v2, CommitOptions{Meta: m2})
	assert.NoError(err)
	h2 := source.Head()

	v3 := types.Number(3)
	m3 := types.NewStruct("Meta", types.StructData{
		"name": types.String("three"),
	})
	source, err = source.Commit(v3, CommitOptions{Meta: m3})
	assert.NoError(err)

	v4 := types.Number(4)
	m4 := types.NewStruct("Meta", types.StructData{
		"name": types.String("three"),
	})
	source, err = source.Commit(v4, CommitOptions{Meta: m4})
	assert.NoError(err)
	h4 := source.Head()

	sink, err = sink.Pull(source.Database(), types.NewRef(h2), 1, nil)
	assert.NoError(err)

	sink, err = sink.Pull(source.Database(), types.NewRef(h4), 1, nil)
	assert.NoError(err)
	assert.True(source.Head().Equals(sink.Head()))
}
Beispiel #7
0
func TestNewCommit(t *testing.T) {
	assert := assert.New(t)

	commitFieldNames := []string{MetaField, ParentsField, ValueField}

	assertTypeEquals := func(e, a *types.Type) {
		assert.True(a.Equals(e), "Actual: %s\nExpected %s", a.Describe(), e.Describe())
	}

	commit := NewCommit(types.Number(1), types.NewSet(), types.EmptyStruct)
	at := commit.Type()
	et := types.MakeStructType("Commit", commitFieldNames, []*types.Type{
		types.EmptyStructType,
		types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))),
		types.NumberType,
	})
	assertTypeEquals(et, at)

	// Commiting another Number
	commit2 := NewCommit(types.Number(2), types.NewSet(types.NewRef(commit)), types.EmptyStruct)
	at2 := commit2.Type()
	et2 := et
	assertTypeEquals(et2, at2)

	// Now commit a String
	commit3 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2)), types.EmptyStruct)
	at3 := commit3.Type()
	et3 := types.MakeStructType("Commit", commitFieldNames, []*types.Type{
		types.EmptyStructType,
		types.MakeSetType(types.MakeRefType(types.MakeStructType("Commit", commitFieldNames, []*types.Type{
			types.EmptyStructType,
			types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))),
			types.MakeUnionType(types.NumberType, types.StringType),
		}))),
		types.StringType,
	})
	assertTypeEquals(et3, at3)

	// Now commit a String with MetaInfo
	meta := types.NewStruct("Meta", types.StructData{"date": types.String("some date"), "number": types.Number(9)})
	metaType := types.MakeStructType("Meta", []string{"date", "number"}, []*types.Type{types.StringType, types.NumberType})
	assertTypeEquals(metaType, meta.Type())
	commit4 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2)), meta)
	at4 := commit4.Type()
	et4 := types.MakeStructType("Commit", commitFieldNames, []*types.Type{
		metaType,
		types.MakeSetType(types.MakeRefType(types.MakeStructType("Commit", commitFieldNames, []*types.Type{
			types.MakeUnionType(types.EmptyStructType, metaType),
			types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))),
			types.MakeUnionType(types.NumberType, types.StringType),
		}))),
		types.StringType,
	})
	assertTypeEquals(et4, at4)
}
Beispiel #8
0
func (suite *LibTestSuite) TestCompositeTypeWithStruct() {
	// {"string": "string",
	//  "list": [false true],
	//  "struct": {"nested": "string"}
	// }
	tstruct := types.NewStruct("", types.StructData{
		"string": types.String("string"),
		"list":   types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
		"struct": types.NewStruct("", types.StructData{
			"nested": types.String("string"),
		}),
	})
	o := NomsValueFromDecodedJSON(map[string]interface{}{
		"string": "string",
		"list":   []interface{}{false, true},
		"struct": map[string]interface{}{"nested": "string"},
	}, true)

	suite.True(tstruct.Equals(o))
}
Beispiel #9
0
func metaInfoForCommit(fileOrUrl, source, comment string) types.Struct {
	date := time.Now().UTC().Format("2006-01-02T15:04:05-0700")
	metaValues := types.StructData{
		"date":    types.String(date),
		fileOrUrl: types.String(source),
	}
	if comment != "" {
		metaValues["comment"] = types.String(comment)
	}
	return types.NewStruct("Meta", metaValues)
}
Beispiel #10
0
func metaInfoForCommit(sourceType, sourceVal, comment string) types.Struct {
	date := time.Now().UTC().Format("2006-01-02T15:04:05-0700")
	metaValues := map[string]types.Value{
		"date": types.String(date),
	}
	if sourceType != "" {
		metaValues[sourceType] = types.String(sourceVal)
	}
	if comment != "" {
		metaValues["comment"] = types.String(comment)
	}
	return types.NewStruct("Meta", metaValues)
}
Beispiel #11
0
// NomsValueFromDecodedJSON takes a generic Go interface{} and recursively
// tries to resolve the types within so that it can build up and return
// a Noms Value with the same structure.
//
// Currently, the only types supported are the Go versions of legal JSON types:
// Primitives:
//  - float64
//  - bool
//  - string
//  - nil
//
// Composites:
//  - []interface{}
//  - map[string]interface{}
func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
	switch o := o.(type) {
	case string:
		return types.String(o)
	case bool:
		return types.Bool(o)
	case float64:
		return types.Number(o)
	case nil:
		return nil
	case []interface{}:
		items := make([]types.Value, 0, len(o))
		for _, v := range o {
			nv := NomsValueFromDecodedJSON(v, useStruct)
			if nv != nil {
				items = append(items, nv)
			}
		}
		return types.NewList(items...)
	case map[string]interface{}:
		var v types.Value
		if useStruct {
			fields := make(map[string]types.Value, len(o))
			for k, v := range o {
				nv := NomsValueFromDecodedJSON(v, useStruct)
				if nv != nil {
					k := types.EscapeStructField(k)
					fields[k] = nv
				}
			}
			v = types.NewStruct("", fields)
		} else {
			kv := make([]types.Value, 0, len(o)*2)
			for k, v := range o {
				nv := NomsValueFromDecodedJSON(v, useStruct)
				if nv != nil {
					kv = append(kv, types.String(k), nv)
				}
			}
			v = types.NewMap(kv...)
		}
		return v

	default:
		d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String())
	}
	return nil
}
Beispiel #12
0
func metaInfoForCommit(date, filePath, nomsPath, comment string) types.Struct {
	fileOrNomsPath := "inputPath"
	path := nomsPath
	if path == "" {
		path = filePath
		fileOrNomsPath = "inputFile"
	}
	metaValues := types.StructData{
		"date":         types.String(date),
		fileOrNomsPath: types.String(path),
	}
	if comment != "" {
		metaValues["comment"] = types.String(comment)
	}
	return types.NewStruct("Meta", metaValues)
}
Beispiel #13
0
func addPerson(ds dataset.Dataset) {
	if flag.NArg() != 4 {
		fmt.Fprintln(os.Stderr, "Not enough arguments for command add-person")
		return
	}

	id, err := strconv.ParseUint(flag.Arg(1), 10, 64)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Invalid person-id: %s", flag.Arg(1))
		return
	}

	np := types.NewStruct("Person", map[string]types.Value{
		"id":    types.Number(id),
		"name":  types.String(flag.Arg(2)),
		"title": types.String(flag.Arg(3)),
	})

	ds.Commit(getPersons(ds).Set(types.Number(id), np))
}
Beispiel #14
0
func (s *nomsLogTestSuite) TestEmptyCommit() {
	str := spec.CreateDatabaseSpecString("ldb", s.LdbDir)
	db, err := spec.GetDatabase(str)
	s.NoError(err)

	ds := dataset.NewDataset(db, "ds1")

	meta := types.NewStruct("Meta", map[string]types.Value{
		"longNameForTest": types.String("Yoo"),
		"test2":           types.String("Hoo"),
	})
	ds, err = ds.Commit(types.String("1"), dataset.CommitOptions{Meta: meta})
	s.NoError(err)

	ds.Commit(types.String("2"), dataset.CommitOptions{})
	db.Close()

	dsSpec := spec.CreateValueSpecString("ldb", s.LdbDir, "ds1")
	res, _ := s.Run(main, []string{"log", "--show-value=false", dsSpec})
	test.EqualsIgnoreHashes(s.T(), metaRes1, res)

	res, _ = s.Run(main, []string{"log", "--show-value=false", "--oneline", dsSpec})
	test.EqualsIgnoreHashes(s.T(), metaRes2, res)
}
Beispiel #15
0
func TestWriteValue(t *testing.T) {
	assert := assert.New(t)
	factory := chunks.NewMemoryStoreFactory()
	defer factory.Shutter()

	router = setupWebServer(factory)
	defer func() { router = nil }()

	testString := "Now, what?"
	authKey = "anauthkeyvalue"

	w := httptest.NewRecorder()
	r, err := newRequest("GET", dbName+constants.RootPath, nil)
	assert.NoError(err)
	router.ServeHTTP(w, r)
	lastRoot := w.Body
	assert.Equal(http.StatusOK, w.Code)

	craftCommit := func(v types.Value) types.Struct {
		return datas.NewCommit(v, types.NewSet(), types.NewStruct("Meta", types.StructData{}))
	}

	tval := craftCommit(types.Bool(true))
	wval := craftCommit(types.String(testString))
	chunk1 := types.EncodeValue(tval, nil)
	chunk2 := types.EncodeValue(wval, nil)
	refMap := types.NewMap(
		types.String("ds1"), types.NewRef(tval),
		types.String("ds2"), types.NewRef(wval))
	chunk3 := types.EncodeValue(refMap, nil)

	body := &bytes.Buffer{}
	// we would use this func, but it's private so use next line instead: serializeHints(body, map[ref.Ref]struct{}{hint: struct{}{}})
	err = binary.Write(body, binary.BigEndian, uint32(0))
	assert.NoError(err)

	chunks.Serialize(chunk1, body)
	chunks.Serialize(chunk2, body)
	chunks.Serialize(chunk3, body)

	w = httptest.NewRecorder()
	r, err = newRequest("POST", dbName+constants.WriteValuePath+"?access_token="+authKey, ioutil.NopCloser(body))
	assert.NoError(err)
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusCreated, w.Code)

	w = httptest.NewRecorder()
	args := fmt.Sprintf("&last=%s&current=%s", lastRoot, types.NewRef(refMap).TargetHash())
	r, _ = newRequest("POST", dbName+constants.RootPath+"?access_token="+authKey+args, ioutil.NopCloser(body))
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusOK, w.Code, string(w.Body.Bytes()))

	whash := wval.Hash()
	hints := map[hash.Hash]struct{}{whash: struct{}{}}
	rdr := buildGetRefsRequestBody(hints)
	r, _ = newRequest("POST", dbName+constants.GetRefsPath, rdr)
	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusOK, w.Code, string(w.Body.Bytes()))

	ms := chunks.NewMemoryStore()
	chunks.Deserialize(w.Body, ms, nil)
	v := types.DecodeValue(ms.Get(whash), datas.NewDatabase(ms))
	assert.Equal(testString, string(v.(types.Struct).Get(datas.ValueField).(types.String)))
}