Example #1
0
// StringToValue takes a piece of data as a string and attempts to convert it to a types.Value of the appropriate types.NomsKind.
func StringToValue(s string, k types.NomsKind) (types.Value, error) {
	switch k {
	case types.NumberKind:
		if s == "" {
			return types.Number(float64(0)), nil
		}
		fval, err := strconv.ParseFloat(s, 64)
		if err != nil {
			return nil, fmt.Errorf("Could not parse '%s' into number (%s)", s, err)
		}
		return types.Number(fval), nil
	case types.BoolKind:
		// TODO: This should probably be configurable.
		switch s {
		case "true", "1", "y", "Y":
			return types.Bool(true), nil
		case "false", "0", "n", "N", "":
			return types.Bool(false), nil
		default:
			return nil, fmt.Errorf("Could not parse '%s' into bool", s)
		}
	case types.StringKind:
		return types.String(s), nil
	default:
		d.PanicIfTrue(true, "Invalid column type kind:", k)
	}
	panic("not reached")
}
Example #2
0
func TestReadToMap(t *testing.T) {
	assert := assert.New(t)
	ds := datas.NewDatabase(chunks.NewMemoryStore())

	dataString := `a,1,true
b,2,false
`
	r := NewCSVReader(bytes.NewBufferString(dataString), ',')

	headers := []string{"A", "B", "C"}
	kinds := KindSlice{types.StringKind, types.NumberKind, types.BoolKind}
	m := ReadToMap(r, headers, 0, kinds, ds)

	assert.Equal(uint64(2), m.Len())
	assert.True(m.Type().Equals(
		types.MakeMapType(types.StringType, types.MakeStructType("", map[string]*types.Type{
			"B": types.NumberType,
			"C": types.BoolType,
		}))))

	assert.True(m.Get(types.String("a")).Equals(types.NewStruct("", map[string]types.Value{
		"B": types.Number(1),
		"C": types.Bool(true),
	})))
	assert.True(m.Get(types.String("b")).Equals(types.NewStruct("", map[string]types.Value{
		"B": types.Number(2),
		"C": types.Bool(false),
	})))
}
func TestHandleWriteValue(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	ds := NewDatabase(cs)

	l := types.NewList(
		ds.WriteValue(types.Bool(true)),
		ds.WriteValue(types.Bool(false)),
	)
	ds.WriteValue(l)

	hint := l.Hash()
	newItem := types.NewEmptyBlob()
	itemChunk := types.EncodeValue(newItem, nil)
	l2 := l.Insert(1, types.NewRef(newItem))
	listChunk := types.EncodeValue(l2, nil)

	body := &bytes.Buffer{}
	serializeHints(body, map[hash.Hash]struct{}{hint: struct{}{}})
	sz := chunks.NewSerializer(body)
	sz.Put(itemChunk)
	sz.Put(listChunk)
	sz.Close()

	w := httptest.NewRecorder()
	HandleWriteValue(w, &http.Request{Body: ioutil.NopCloser(body), Method: "POST"}, params{}, cs)

	if assert.Equal(http.StatusCreated, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
		ds2 := NewDatabase(cs)
		v := ds2.ReadValue(l2.Hash())
		if assert.NotNil(v) {
			assert.True(v.Equals(l2), "%+v != %+v", v, l2)
		}
	}
}
func TestHandleWriteValueBackpressure(t *testing.T) {
	assert := assert.New(t)
	cs := &backpressureCS{ChunkStore: chunks.NewMemoryStore()}
	ds := NewDatabase(cs)

	l := types.NewList(
		ds.WriteValue(types.Bool(true)),
		ds.WriteValue(types.Bool(false)),
	)
	ds.WriteValue(l)

	hint := l.Hash()
	newItem := types.NewEmptyBlob()
	itemChunk := types.EncodeValue(newItem, nil)
	l2 := l.Insert(1, types.NewRef(newItem))
	listChunk := types.EncodeValue(l2, nil)

	body := &bytes.Buffer{}
	serializeHints(body, map[hash.Hash]struct{}{hint: struct{}{}})
	sz := chunks.NewSerializer(body)
	sz.Put(itemChunk)
	sz.Put(listChunk)
	sz.Close()

	w := httptest.NewRecorder()
	HandleWriteValue(w, &http.Request{Body: ioutil.NopCloser(body), Method: "POST"}, params{}, cs)

	if assert.Equal(httpStatusTooManyRequests, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
		hashes := deserializeHashes(w.Body)
		assert.Len(hashes, 1)
		assert.Equal(l2.Hash(), hashes[0])
	}
}
Example #5
0
func TestReadToList(t *testing.T) {
	assert := assert.New(t)
	ds := datas.NewDatabase(chunks.NewMemoryStore())

	dataString := `a,1,true
b,2,false
`
	r := NewCSVReader(bytes.NewBufferString(dataString), ',')

	headers := []string{"A", "B", "C"}
	kinds := KindSlice{types.StringKind, types.NumberKind, types.BoolKind}
	l, typ := ReadToList(r, "test", headers, kinds, ds)

	assert.Equal(uint64(2), l.Len())

	assert.Equal(types.StructKind, typ.Kind())

	desc, ok := typ.Desc.(types.StructDesc)
	assert.True(ok)
	assert.Equal(desc.Len(), 3)
	assert.Equal(types.StringKind, desc.Field("A").Kind())
	assert.Equal(types.NumberKind, desc.Field("B").Kind())
	assert.Equal(types.BoolKind, desc.Field("C").Kind())

	assert.True(l.Get(0).(types.Struct).Get("A").Equals(types.String("a")))
	assert.True(l.Get(1).(types.Struct).Get("A").Equals(types.String("b")))

	assert.True(l.Get(0).(types.Struct).Get("B").Equals(types.Number(1)))
	assert.True(l.Get(1).(types.Struct).Get("B").Equals(types.Number(2)))

	assert.True(l.Get(0).(types.Struct).Get("C").Equals(types.Bool(true)))
	assert.True(l.Get(1).(types.Struct).Get("C").Equals(types.Bool(false)))
}
Example #6
0
func (suite *WalkAllTestSuite) TestWalkComposites() {
	suite.walkWorker(suite.storeAndRef(types.NewList()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewList(types.Bool(false), types.Number(8))), 4)
	suite.walkWorker(suite.storeAndRef(types.NewSet()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewSet(types.Bool(false), types.Number(8))), 4)
	suite.walkWorker(suite.storeAndRef(types.NewMap()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewMap(types.Number(8), types.Bool(true), types.Number(0), types.Bool(false))), 6)
}
Example #7
0
func (suite *DatabaseSuite) TestReadWriteCache() {
	var v types.Value = types.Bool(true)
	suite.NotEqual(hash.Hash{}, suite.ds.WriteValue(v))
	r := suite.ds.WriteValue(v).TargetHash()
	commit := NewCommit(v, types.NewSet(), types.EmptyStruct)
	newDs, err := suite.ds.Commit("foo", commit)
	suite.NoError(err)
	suite.Equal(1, suite.cs.Writes-writesOnCommit)

	v = newDs.ReadValue(r)
	suite.True(v.Equals(types.Bool(true)))
}
Example #8
0
func TestMemDatabase(t *testing.T) {
	assert := assert.New(t)

	spec := "mem"
	sp, err := ParseDatabaseSpec(spec)
	assert.NoError(err)
	store, err := sp.Database()
	assert.NoError(err)
	r := store.WriteValue(types.Bool(true))

	assert.NoError(err)
	assert.Equal(types.Bool(true), store.ReadValue(r.TargetHash()))
}
Example #9
0
func TestBooleanStrings(t *testing.T) {
	assert := assert.New(t)
	ds := datas.NewDatabase(chunks.NewMemoryStore())
	dataString := "true,false\n1,0\ny,n\nY,N\nY,\n"
	r := NewCSVReader(bytes.NewBufferString(dataString), ',')
	headers := []string{"T", "F"}
	kinds := KindSlice{types.BoolKind, types.BoolKind}

	l, _ := ReadToList(r, "test", headers, kinds, ds)
	assert.Equal(uint64(5), l.Len())
	for i := uint64(0); i < l.Len(); i++ {
		row := l.Get(i).(types.Struct)
		assert.True(types.Bool(true).Equals(row.Get("T")))
		assert.True(types.Bool(false).Equals(row.Get("F")))
	}
}
Example #10
0
func createStruct(i uint64) types.Value {
	return types.NewStructWithType(structType, types.ValueSlice{
		types.Bool(i%2 == 0),
		types.Number(i),
		types.String(fmt.Sprintf("i am a 55 bytes............................%12d", i)),
	})
}
Example #11
0
func createStruct(i uint64) types.Value {
	return types.NewStructWithType(structType, map[string]types.Value{
		"str":  types.String(fmt.Sprintf("i am a 55 bytes............................%12d", i)),
		"num":  types.Number(i),
		"bool": types.Bool(i%2 == 0),
	})
}
func TestWriteValue(t *testing.T) {
	assert := assert.New(t)
	factory := chunks.NewMemoryStoreFactory()
	defer factory.Shutter()

	router = setupWebServer(factory)
	defer func() { router = nil }()

	testString := "Now, what?"
	authKey = "anauthkeyvalue"

	w := httptest.NewRecorder()
	r, err := newRequest("GET", dbName+constants.RootPath, nil)
	assert.NoError(err)
	router.ServeHTTP(w, r)
	lastRoot := w.Body
	assert.Equal(http.StatusOK, w.Code)

	tval := types.Bool(true)
	wval := types.String(testString)
	chunk1 := types.EncodeValue(tval, nil)
	chunk2 := types.EncodeValue(wval, nil)
	refList := types.NewList(types.NewRef(tval), types.NewRef(wval))
	chunk3 := types.EncodeValue(refList, nil)

	body := &bytes.Buffer{}
	// we would use this func, but it's private so use next line instead: serializeHints(body, map[ref.Ref]struct{}{hint: struct{}{}})
	err = binary.Write(body, binary.BigEndian, uint32(0))
	assert.NoError(err)

	chunks.Serialize(chunk1, body)
	chunks.Serialize(chunk2, body)
	chunks.Serialize(chunk3, body)

	w = httptest.NewRecorder()
	r, err = newRequest("POST", dbName+constants.WriteValuePath+"?access_token="+authKey, ioutil.NopCloser(body))
	assert.NoError(err)
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusCreated, w.Code)

	w = httptest.NewRecorder()
	args := fmt.Sprintf("&last=%s&current=%s", lastRoot, types.NewRef(refList).TargetHash())
	r, _ = newRequest("POST", dbName+constants.RootPath+"?access_token="+authKey+args, ioutil.NopCloser(body))
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusOK, w.Code)

	whash := wval.Hash()
	hints := map[hash.Hash]struct{}{whash: struct{}{}}
	rdr := buildGetRefsRequestBody(hints)
	r, _ = newRequest("POST", dbName+constants.GetRefsPath, rdr)
	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusOK, w.Code)

	ms := chunks.NewMemoryStore()
	chunks.Deserialize(w.Body, ms, nil)
	v := types.DecodeValue(ms.Get(whash), datas.NewDatabase(ms))
	assert.Equal(testString, string(v.(types.String)))
}
Example #13
0
func TestValidateRef(t *testing.T) {
	ds := createTestDataset("test")
	b := types.Bool(true)
	r := ds.Database().WriteValue(b)

	assert.Panics(t, func() { ds.validateRefAsCommit(r) })
	assert.Panics(t, func() { ds.validateRefAsCommit(types.NewRef(b)) })
}
Example #14
0
func (suite *LibTestSuite) TestCompositeTypeWithStruct() {
	// {"string": "string",
	//  "list": [false true],
	//  "struct": {"nested": "string"}
	// }
	tstruct := types.NewStruct("", types.StructData{
		"string": types.String("string"),
		"list":   types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
		"struct": types.NewStruct("", types.StructData{
			"nested": types.String("string"),
		}),
	})
	o := NomsValueFromDecodedJSON(map[string]interface{}{
		"string": "string",
		"list":   []interface{}{false, true},
		"struct": map[string]interface{}{"nested": "string"},
	}, true)

	suite.True(tstruct.Equals(o))
}
Example #15
0
func (suite *WalkAllTestSuite) TestWalkNestedComposites() {
	cs := chunks.NewMemoryStore()
	suite.walkWorker(suite.storeAndRef(types.NewList(suite.NewSet(cs), types.Number(8))), 5)
	suite.walkWorker(suite.storeAndRef(types.NewSet(suite.NewList(cs), suite.NewSet(cs))), 6)
	// {"string": "string",
	//  "list": [false true],
	//  "map": {"nested": "string"}
	//  "mtlist": []
	//  "set": [5 7 8]
	//  []: "wow"
	// }
	nested := types.NewMap(
		types.String("string"), types.String("string"),
		types.String("list"), suite.NewList(cs, types.Bool(false), types.Bool(true)),
		types.String("map"), suite.NewMap(cs, types.String("nested"), types.String("string")),
		types.String("mtlist"), suite.NewList(cs),
		types.String("set"), suite.NewSet(cs, types.Number(5), types.Number(7), types.Number(8)),
		suite.NewList(cs), types.String("wow"), // note that the dupe list chunk is skipped
	)
	suite.walkWorker(suite.storeAndRef(nested), 25)
}
Example #16
0
func (suite *LibTestSuite) TestCompositeTypes() {
	// [false true]
	suite.EqualValues(
		types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
		NomsValueFromDecodedJSON([]interface{}{false, true}, false))

	// [[false true]]
	suite.EqualValues(
		types.NewList().Append(
			types.NewList().Append(types.Bool(false)).Append(types.Bool(true))),
		NomsValueFromDecodedJSON([]interface{}{[]interface{}{false, true}}, false))

	// {"string": "string",
	//  "list": [false true],
	//  "map": {"nested": "string"}
	// }
	m := types.NewMap(
		types.String("string"),
		types.String("string"),
		types.String("list"),
		types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
		types.String("map"),
		types.NewMap(
			types.String("nested"),
			types.String("string")))
	o := NomsValueFromDecodedJSON(map[string]interface{}{
		"string": "string",
		"list":   []interface{}{false, true},
		"map":    map[string]interface{}{"nested": "string"},
	}, false)

	suite.True(m.Equals(o))
}
Example #17
0
func (suite *DatabaseSuite) TestReadWriteCachePersists() {
	var err error
	var v types.Value = types.Bool(true)
	suite.NotEqual(hash.Hash{}, suite.ds.WriteValue(v))
	r := suite.ds.WriteValue(v)
	commit := NewCommit(v, types.NewSet(), types.EmptyStruct)
	suite.ds, err = suite.ds.Commit("foo", commit)
	suite.NoError(err)
	suite.Equal(1, suite.cs.Writes-writesOnCommit)

	newCommit := NewCommit(r, types.NewSet(types.NewRef(commit)), types.EmptyStruct)
	suite.ds, err = suite.ds.Commit("foo", newCommit)
	suite.NoError(err)
}
Example #18
0
// NomsValueFromDecodedJSON takes a generic Go interface{} and recursively
// tries to resolve the types within so that it can build up and return
// a Noms Value with the same structure.
//
// Currently, the only types supported are the Go versions of legal JSON types:
// Primitives:
//  - float64
//  - bool
//  - string
//  - nil
//
// Composites:
//  - []interface{}
//  - map[string]interface{}
func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
	switch o := o.(type) {
	case string:
		return types.String(o)
	case bool:
		return types.Bool(o)
	case float64:
		return types.Number(o)
	case nil:
		return nil
	case []interface{}:
		items := make([]types.Value, 0, len(o))
		for _, v := range o {
			nv := NomsValueFromDecodedJSON(v, useStruct)
			if nv != nil {
				items = append(items, nv)
			}
		}
		return types.NewList(items...)
	case map[string]interface{}:
		var v types.Value
		if useStruct {
			fields := make(map[string]types.Value, len(o))
			for k, v := range o {
				nv := NomsValueFromDecodedJSON(v, useStruct)
				if nv != nil {
					k := types.EscapeStructField(k)
					fields[k] = nv
				}
			}
			v = types.NewStruct("", fields)
		} else {
			kv := make([]types.Value, 0, len(o)*2)
			for k, v := range o {
				nv := NomsValueFromDecodedJSON(v, useStruct)
				if nv != nil {
					kv = append(kv, types.String(k), nv)
				}
			}
			v = types.NewMap(kv...)
		}
		return v

	default:
		d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String())
	}
	return nil
}
Example #19
0
func TestDefaults(t *testing.T) {
	assert := assert.New(t)
	ds := datas.NewDatabase(chunks.NewMemoryStore())
	dataString := "42,,,\n"
	r := NewCSVReader(bytes.NewBufferString(dataString), ',')
	headers := []string{"A", "B", "C", "D"}
	kinds := KindSlice{types.NumberKind, types.NumberKind, types.BoolKind, types.StringKind}

	l, _ := ReadToList(r, "test", headers, kinds, ds)
	assert.Equal(uint64(1), l.Len())
	row := l.Get(0).(types.Struct)
	assert.Equal(types.Number(42), row.Get("A"))
	assert.Equal(types.Number(0), row.Get("B"))
	assert.Equal(types.Bool(false), row.Get("C"))
	assert.Equal(types.String(""), row.Get("D"))
}
Example #20
0
// StringToType takes a piece of data as a string and attempts to convert it to a types.Value of the appropriate types.NomsKind.
func StringToType(s string, k types.NomsKind) types.Value {
	switch k {
	case types.NumberKind:
		fval, err := strconv.ParseFloat(s, 64)
		d.Chk.NoError(err)
		return types.Number(fval)
	case types.BoolKind:
		bval, err := strconv.ParseBool(s)
		d.Chk.NoError(err)
		return types.Bool(bval)
	case types.StringKind:
		return types.String(s)
	default:
		d.Exp.Fail("Invalid column type kind:", k)
	}
	panic("not reached")
}
Example #21
0
func (suite *LibTestSuite) TestPrimitiveTypes() {
	suite.EqualValues(types.String("expected"), NomsValueFromDecodedJSON("expected", false))
	suite.EqualValues(types.Bool(false), NomsValueFromDecodedJSON(false, false))
	suite.EqualValues(types.Number(1.7), NomsValueFromDecodedJSON(1.7, false))
	suite.False(NomsValueFromDecodedJSON(1.7, false).Equals(types.Bool(true)))
}
Example #22
0
func (suite *DatabaseSuite) TestWriteRefToNonexistentValue() {
	suite.Panics(func() { suite.ds.WriteValue(types.NewRef(types.Bool(true))) })
}