// StringToValue takes a piece of data as a string and attempts to convert it to a types.Value of the appropriate types.NomsKind.
func StringToValue(s string, k types.NomsKind) (types.Value, error) {
switch k {
case types.NumberKind:
if s == "" {
return types.Number(float64(0)), nil
}
fval, err := strconv.ParseFloat(s, 64)
if err != nil {
return nil, fmt.Errorf("Could not parse '%s' into number (%s)", s, err)
}
return types.Number(fval), nil
case types.BoolKind:
// TODO: This should probably be configurable.
switch s {
case "true", "1", "y", "Y":
return types.Bool(true), nil
case "false", "0", "n", "N", "":
return types.Bool(false), nil
default:
return nil, fmt.Errorf("Could not parse '%s' into bool", s)
}
case types.StringKind:
return types.String(s), nil
default:
d.PanicIfTrue(true, "Invalid column type kind:", k)
}
panic("not reached")
}
func TestReadToMap(t *testing.T) {
assert := assert.New(t)
ds := datas.NewDatabase(chunks.NewMemoryStore())
dataString := `a,1,true
b,2,false
`
r := NewCSVReader(bytes.NewBufferString(dataString), ',')
headers := []string{"A", "B", "C"}
kinds := KindSlice{types.StringKind, types.NumberKind, types.BoolKind}
m := ReadToMap(r, headers, 0, kinds, ds)
assert.Equal(uint64(2), m.Len())
assert.True(m.Type().Equals(
types.MakeMapType(types.StringType, types.MakeStructType("", map[string]*types.Type{
"B": types.NumberType,
"C": types.BoolType,
}))))
assert.True(m.Get(types.String("a")).Equals(types.NewStruct("", map[string]types.Value{
"B": types.Number(1),
"C": types.Bool(true),
})))
assert.True(m.Get(types.String("b")).Equals(types.NewStruct("", map[string]types.Value{
"B": types.Number(2),
"C": types.Bool(false),
})))
}
func TestHandleWriteValue(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
ds := NewDatabase(cs)
l := types.NewList(
ds.WriteValue(types.Bool(true)),
ds.WriteValue(types.Bool(false)),
)
ds.WriteValue(l)
hint := l.Hash()
newItem := types.NewEmptyBlob()
itemChunk := types.EncodeValue(newItem, nil)
l2 := l.Insert(1, types.NewRef(newItem))
listChunk := types.EncodeValue(l2, nil)
body := &bytes.Buffer{}
serializeHints(body, map[hash.Hash]struct{}{hint: struct{}{}})
sz := chunks.NewSerializer(body)
sz.Put(itemChunk)
sz.Put(listChunk)
sz.Close()
w := httptest.NewRecorder()
HandleWriteValue(w, &http.Request{Body: ioutil.NopCloser(body), Method: "POST"}, params{}, cs)
if assert.Equal(http.StatusCreated, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
ds2 := NewDatabase(cs)
v := ds2.ReadValue(l2.Hash())
if assert.NotNil(v) {
assert.True(v.Equals(l2), "%+v != %+v", v, l2)
}
}
}
func TestHandleWriteValueBackpressure(t *testing.T) {
assert := assert.New(t)
cs := &backpressureCS{ChunkStore: chunks.NewMemoryStore()}
ds := NewDatabase(cs)
l := types.NewList(
ds.WriteValue(types.Bool(true)),
ds.WriteValue(types.Bool(false)),
)
ds.WriteValue(l)
hint := l.Hash()
newItem := types.NewEmptyBlob()
itemChunk := types.EncodeValue(newItem, nil)
l2 := l.Insert(1, types.NewRef(newItem))
listChunk := types.EncodeValue(l2, nil)
body := &bytes.Buffer{}
serializeHints(body, map[hash.Hash]struct{}{hint: struct{}{}})
sz := chunks.NewSerializer(body)
sz.Put(itemChunk)
sz.Put(listChunk)
sz.Close()
w := httptest.NewRecorder()
HandleWriteValue(w, &http.Request{Body: ioutil.NopCloser(body), Method: "POST"}, params{}, cs)
if assert.Equal(httpStatusTooManyRequests, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
hashes := deserializeHashes(w.Body)
assert.Len(hashes, 1)
assert.Equal(l2.Hash(), hashes[0])
}
}
func TestReadToList(t *testing.T) {
assert := assert.New(t)
ds := datas.NewDatabase(chunks.NewMemoryStore())
dataString := `a,1,true
b,2,false
`
r := NewCSVReader(bytes.NewBufferString(dataString), ',')
headers := []string{"A", "B", "C"}
kinds := KindSlice{types.StringKind, types.NumberKind, types.BoolKind}
l, typ := ReadToList(r, "test", headers, kinds, ds)
assert.Equal(uint64(2), l.Len())
assert.Equal(types.StructKind, typ.Kind())
desc, ok := typ.Desc.(types.StructDesc)
assert.True(ok)
assert.Equal(desc.Len(), 3)
assert.Equal(types.StringKind, desc.Field("A").Kind())
assert.Equal(types.NumberKind, desc.Field("B").Kind())
assert.Equal(types.BoolKind, desc.Field("C").Kind())
assert.True(l.Get(0).(types.Struct).Get("A").Equals(types.String("a")))
assert.True(l.Get(1).(types.Struct).Get("A").Equals(types.String("b")))
assert.True(l.Get(0).(types.Struct).Get("B").Equals(types.Number(1)))
assert.True(l.Get(1).(types.Struct).Get("B").Equals(types.Number(2)))
assert.True(l.Get(0).(types.Struct).Get("C").Equals(types.Bool(true)))
assert.True(l.Get(1).(types.Struct).Get("C").Equals(types.Bool(false)))
}
func (suite *WalkAllTestSuite) TestWalkComposites() {
suite.walkWorker(suite.storeAndRef(types.NewList()), 2)
suite.walkWorker(suite.storeAndRef(types.NewList(types.Bool(false), types.Number(8))), 4)
suite.walkWorker(suite.storeAndRef(types.NewSet()), 2)
suite.walkWorker(suite.storeAndRef(types.NewSet(types.Bool(false), types.Number(8))), 4)
suite.walkWorker(suite.storeAndRef(types.NewMap()), 2)
suite.walkWorker(suite.storeAndRef(types.NewMap(types.Number(8), types.Bool(true), types.Number(0), types.Bool(false))), 6)
}
func (suite *DatabaseSuite) TestReadWriteCache() {
var v types.Value = types.Bool(true)
suite.NotEqual(hash.Hash{}, suite.ds.WriteValue(v))
r := suite.ds.WriteValue(v).TargetHash()
commit := NewCommit(v, types.NewSet(), types.EmptyStruct)
newDs, err := suite.ds.Commit("foo", commit)
suite.NoError(err)
suite.Equal(1, suite.cs.Writes-writesOnCommit)
v = newDs.ReadValue(r)
suite.True(v.Equals(types.Bool(true)))
}
func TestMemDatabase(t *testing.T) {
assert := assert.New(t)
spec := "mem"
sp, err := ParseDatabaseSpec(spec)
assert.NoError(err)
store, err := sp.Database()
assert.NoError(err)
r := store.WriteValue(types.Bool(true))
assert.NoError(err)
assert.Equal(types.Bool(true), store.ReadValue(r.TargetHash()))
}
func TestBooleanStrings(t *testing.T) {
assert := assert.New(t)
ds := datas.NewDatabase(chunks.NewMemoryStore())
dataString := "true,false\n1,0\ny,n\nY,N\nY,\n"
r := NewCSVReader(bytes.NewBufferString(dataString), ',')
headers := []string{"T", "F"}
kinds := KindSlice{types.BoolKind, types.BoolKind}
l, _ := ReadToList(r, "test", headers, kinds, ds)
assert.Equal(uint64(5), l.Len())
for i := uint64(0); i < l.Len(); i++ {
row := l.Get(i).(types.Struct)
assert.True(types.Bool(true).Equals(row.Get("T")))
assert.True(types.Bool(false).Equals(row.Get("F")))
}
}
func createStruct(i uint64) types.Value {
return types.NewStructWithType(structType, types.ValueSlice{
types.Bool(i%2 == 0),
types.Number(i),
types.String(fmt.Sprintf("i am a 55 bytes............................%12d", i)),
})
}
func createStruct(i uint64) types.Value {
return types.NewStructWithType(structType, map[string]types.Value{
"str": types.String(fmt.Sprintf("i am a 55 bytes............................%12d", i)),
"num": types.Number(i),
"bool": types.Bool(i%2 == 0),
})
}
func TestWriteValue(t *testing.T) {
assert := assert.New(t)
factory := chunks.NewMemoryStoreFactory()
defer factory.Shutter()
router = setupWebServer(factory)
defer func() { router = nil }()
testString := "Now, what?"
authKey = "anauthkeyvalue"
w := httptest.NewRecorder()
r, err := newRequest("GET", dbName+constants.RootPath, nil)
assert.NoError(err)
router.ServeHTTP(w, r)
lastRoot := w.Body
assert.Equal(http.StatusOK, w.Code)
tval := types.Bool(true)
wval := types.String(testString)
chunk1 := types.EncodeValue(tval, nil)
chunk2 := types.EncodeValue(wval, nil)
refList := types.NewList(types.NewRef(tval), types.NewRef(wval))
chunk3 := types.EncodeValue(refList, nil)
body := &bytes.Buffer{}
// we would use this func, but it's private so use next line instead: serializeHints(body, map[ref.Ref]struct{}{hint: struct{}{}})
err = binary.Write(body, binary.BigEndian, uint32(0))
assert.NoError(err)
chunks.Serialize(chunk1, body)
chunks.Serialize(chunk2, body)
chunks.Serialize(chunk3, body)
w = httptest.NewRecorder()
r, err = newRequest("POST", dbName+constants.WriteValuePath+"?access_token="+authKey, ioutil.NopCloser(body))
assert.NoError(err)
router.ServeHTTP(w, r)
assert.Equal(http.StatusCreated, w.Code)
w = httptest.NewRecorder()
args := fmt.Sprintf("&last=%s¤t=%s", lastRoot, types.NewRef(refList).TargetHash())
r, _ = newRequest("POST", dbName+constants.RootPath+"?access_token="+authKey+args, ioutil.NopCloser(body))
router.ServeHTTP(w, r)
assert.Equal(http.StatusOK, w.Code)
whash := wval.Hash()
hints := map[hash.Hash]struct{}{whash: struct{}{}}
rdr := buildGetRefsRequestBody(hints)
r, _ = newRequest("POST", dbName+constants.GetRefsPath, rdr)
r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, r)
assert.Equal(http.StatusOK, w.Code)
ms := chunks.NewMemoryStore()
chunks.Deserialize(w.Body, ms, nil)
v := types.DecodeValue(ms.Get(whash), datas.NewDatabase(ms))
assert.Equal(testString, string(v.(types.String)))
}
func TestValidateRef(t *testing.T) {
ds := createTestDataset("test")
b := types.Bool(true)
r := ds.Database().WriteValue(b)
assert.Panics(t, func() { ds.validateRefAsCommit(r) })
assert.Panics(t, func() { ds.validateRefAsCommit(types.NewRef(b)) })
}
func (suite *LibTestSuite) TestCompositeTypeWithStruct() {
// {"string": "string",
// "list": [false true],
// "struct": {"nested": "string"}
// }
tstruct := types.NewStruct("", types.StructData{
"string": types.String("string"),
"list": types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
"struct": types.NewStruct("", types.StructData{
"nested": types.String("string"),
}),
})
o := NomsValueFromDecodedJSON(map[string]interface{}{
"string": "string",
"list": []interface{}{false, true},
"struct": map[string]interface{}{"nested": "string"},
}, true)
suite.True(tstruct.Equals(o))
}
func (suite *WalkAllTestSuite) TestWalkNestedComposites() {
cs := chunks.NewMemoryStore()
suite.walkWorker(suite.storeAndRef(types.NewList(suite.NewSet(cs), types.Number(8))), 5)
suite.walkWorker(suite.storeAndRef(types.NewSet(suite.NewList(cs), suite.NewSet(cs))), 6)
// {"string": "string",
// "list": [false true],
// "map": {"nested": "string"}
// "mtlist": []
// "set": [5 7 8]
// []: "wow"
// }
nested := types.NewMap(
types.String("string"), types.String("string"),
types.String("list"), suite.NewList(cs, types.Bool(false), types.Bool(true)),
types.String("map"), suite.NewMap(cs, types.String("nested"), types.String("string")),
types.String("mtlist"), suite.NewList(cs),
types.String("set"), suite.NewSet(cs, types.Number(5), types.Number(7), types.Number(8)),
suite.NewList(cs), types.String("wow"), // note that the dupe list chunk is skipped
)
suite.walkWorker(suite.storeAndRef(nested), 25)
}
func (suite *LibTestSuite) TestCompositeTypes() {
// [false true]
suite.EqualValues(
types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
NomsValueFromDecodedJSON([]interface{}{false, true}, false))
// [[false true]]
suite.EqualValues(
types.NewList().Append(
types.NewList().Append(types.Bool(false)).Append(types.Bool(true))),
NomsValueFromDecodedJSON([]interface{}{[]interface{}{false, true}}, false))
// {"string": "string",
// "list": [false true],
// "map": {"nested": "string"}
// }
m := types.NewMap(
types.String("string"),
types.String("string"),
types.String("list"),
types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
types.String("map"),
types.NewMap(
types.String("nested"),
types.String("string")))
o := NomsValueFromDecodedJSON(map[string]interface{}{
"string": "string",
"list": []interface{}{false, true},
"map": map[string]interface{}{"nested": "string"},
}, false)
suite.True(m.Equals(o))
}
func (suite *DatabaseSuite) TestReadWriteCachePersists() {
var err error
var v types.Value = types.Bool(true)
suite.NotEqual(hash.Hash{}, suite.ds.WriteValue(v))
r := suite.ds.WriteValue(v)
commit := NewCommit(v, types.NewSet(), types.EmptyStruct)
suite.ds, err = suite.ds.Commit("foo", commit)
suite.NoError(err)
suite.Equal(1, suite.cs.Writes-writesOnCommit)
newCommit := NewCommit(r, types.NewSet(types.NewRef(commit)), types.EmptyStruct)
suite.ds, err = suite.ds.Commit("foo", newCommit)
suite.NoError(err)
}
// NomsValueFromDecodedJSON takes a generic Go interface{} and recursively
// tries to resolve the types within so that it can build up and return
// a Noms Value with the same structure.
//
// Currently, the only types supported are the Go versions of legal JSON types:
// Primitives:
// - float64
// - bool
// - string
// - nil
//
// Composites:
// - []interface{}
// - map[string]interface{}
func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
switch o := o.(type) {
case string:
return types.String(o)
case bool:
return types.Bool(o)
case float64:
return types.Number(o)
case nil:
return nil
case []interface{}:
items := make([]types.Value, 0, len(o))
for _, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
items = append(items, nv)
}
}
return types.NewList(items...)
case map[string]interface{}:
var v types.Value
if useStruct {
fields := make(map[string]types.Value, len(o))
for k, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
k := types.EscapeStructField(k)
fields[k] = nv
}
}
v = types.NewStruct("", fields)
} else {
kv := make([]types.Value, 0, len(o)*2)
for k, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
kv = append(kv, types.String(k), nv)
}
}
v = types.NewMap(kv...)
}
return v
default:
d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String())
}
return nil
}
func TestDefaults(t *testing.T) {
assert := assert.New(t)
ds := datas.NewDatabase(chunks.NewMemoryStore())
dataString := "42,,,\n"
r := NewCSVReader(bytes.NewBufferString(dataString), ',')
headers := []string{"A", "B", "C", "D"}
kinds := KindSlice{types.NumberKind, types.NumberKind, types.BoolKind, types.StringKind}
l, _ := ReadToList(r, "test", headers, kinds, ds)
assert.Equal(uint64(1), l.Len())
row := l.Get(0).(types.Struct)
assert.Equal(types.Number(42), row.Get("A"))
assert.Equal(types.Number(0), row.Get("B"))
assert.Equal(types.Bool(false), row.Get("C"))
assert.Equal(types.String(""), row.Get("D"))
}
// StringToType takes a piece of data as a string and attempts to convert it to a types.Value of the appropriate types.NomsKind.
func StringToType(s string, k types.NomsKind) types.Value {
switch k {
case types.NumberKind:
fval, err := strconv.ParseFloat(s, 64)
d.Chk.NoError(err)
return types.Number(fval)
case types.BoolKind:
bval, err := strconv.ParseBool(s)
d.Chk.NoError(err)
return types.Bool(bval)
case types.StringKind:
return types.String(s)
default:
d.Exp.Fail("Invalid column type kind:", k)
}
panic("not reached")
}
func (suite *LibTestSuite) TestPrimitiveTypes() {
suite.EqualValues(types.String("expected"), NomsValueFromDecodedJSON("expected", false))
suite.EqualValues(types.Bool(false), NomsValueFromDecodedJSON(false, false))
suite.EqualValues(types.Number(1.7), NomsValueFromDecodedJSON(1.7, false))
suite.False(NomsValueFromDecodedJSON(1.7, false).Equals(types.Bool(true)))
}
func (suite *DatabaseSuite) TestWriteRefToNonexistentValue() {
suite.Panics(func() { suite.ds.WriteValue(types.NewRef(types.Bool(true))) })
}