func TestReadToMap(t *testing.T) {
assert := assert.New(t)
ds := datas.NewDatabase(chunks.NewMemoryStore())
dataString := `a,1,true
b,2,false
`
r := NewCSVReader(bytes.NewBufferString(dataString), ',')
headers := []string{"A", "B", "C"}
kinds := KindSlice{types.StringKind, types.NumberKind, types.BoolKind}
m := ReadToMap(r, headers, 0, kinds, ds)
assert.Equal(uint64(2), m.Len())
assert.True(m.Type().Equals(
types.MakeMapType(types.StringType, types.MakeStructType("", map[string]*types.Type{
"B": types.NumberType,
"C": types.BoolType,
}))))
assert.True(m.Get(types.String("a")).Equals(types.NewStruct("", map[string]types.Value{
"B": types.Number(1),
"C": types.Bool(true),
})))
assert.True(m.Get(types.String("b")).Equals(types.NewStruct("", map[string]types.Value{
"B": types.Number(2),
"C": types.Bool(false),
})))
}
func TestCommitWithoutMetaField(t *testing.T) {
assert := assert.New(t)
metaCommit := types.NewStruct("Commit", types.StructData{
"value": types.Number(9),
"parents": types.NewSet(),
"meta": types.EmptyStruct,
})
assert.True(IsCommitType(metaCommit.Type()))
noMetaCommit := types.NewStruct("Commit", types.StructData{
"value": types.Number(9),
"parents": types.NewSet(),
})
assert.False(IsCommitType(noMetaCommit.Type()))
}
func TestHandlePostRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
vs := types.NewValueStore(types.NewBatchStoreAdaptor(cs))
commit := NewCommit(types.String("head"), types.NewSet(), types.NewStruct("Meta", types.StructData{}))
newHead := types.NewMap(types.String("dataset1"), vs.WriteValue(commit))
chnx := []chunks.Chunk{
chunks.NewChunk([]byte("abc")),
types.EncodeValue(newHead, nil),
}
err := cs.PutMany(chnx)
assert.NoError(err)
// First attempt should fail, as 'last' won't match.
u := &url.URL{}
queryParams := url.Values{}
queryParams.Add("last", chnx[0].Hash().String())
queryParams.Add("current", chnx[1].Hash().String())
u.RawQuery = queryParams.Encode()
url := u.String()
w := httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
// Now, update the root manually to 'last' and try again.
assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
w = httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func createStruct(name string, kv ...interface{}) types.Struct {
fields := types.StructData{}
for i := 0; i < len(kv); i += 2 {
fields[kv[i].(string)] = valToTypesValue(kv[i+1])
}
return types.NewStruct(name, fields)
}
func (s *testSuite) TestCSVImporterToMap() {
input, err := ioutil.TempFile(s.TempDir, "")
d.Chk.NoError(err)
defer input.Close()
defer os.Remove(input.Name())
_, err = input.WriteString("a,b,c\n")
d.Chk.NoError(err)
for i := 0; i < 20; i++ {
_, err = input.WriteString(fmt.Sprintf("a%d,%d,%d\n", i, i, i*2))
d.Chk.NoError(err)
}
_, err = input.Seek(0, 0)
d.Chk.NoError(err)
setName := "csv"
dataspec := test_util.CreateValueSpecString("ldb", s.LdbDir, setName)
out := s.Run(main, []string{"-no-progress", "-column-types", "String,Number,Number", "-dest-type", "map:1", dataspec, input.Name()})
s.Equal("", out)
cs := chunks.NewLevelDBStore(s.LdbDir, "", 1, false)
ds := dataset.NewDataset(datas.NewDatabase(cs), setName)
defer ds.Database().Close()
defer os.RemoveAll(s.LdbDir)
m := ds.HeadValue().(types.Map)
s.Equal(uint64(20), m.Len())
for i := 0; i < 20; i++ {
m.Get(types.Number(i)).(types.Struct).Equals(types.NewStruct("", map[string]types.Value{
"a": types.String(fmt.Sprintf("a%d", i)),
"c": types.Number(i * 2),
}))
}
}
func TestPullWithMeta(t *testing.T) {
assert := assert.New(t)
sink := createTestDataset("sink")
source := createTestDataset("source")
v1 := types.Number(1)
m1 := types.NewStruct("Meta", types.StructData{
"name": types.String("one"),
})
source, err := source.Commit(v1, CommitOptions{Meta: m1})
assert.NoError(err)
v2 := types.Number(2)
m2 := types.NewStruct("Meta", types.StructData{
"name": types.String("two"),
})
source, err = source.Commit(v2, CommitOptions{Meta: m2})
assert.NoError(err)
h2 := source.Head()
v3 := types.Number(3)
m3 := types.NewStruct("Meta", types.StructData{
"name": types.String("three"),
})
source, err = source.Commit(v3, CommitOptions{Meta: m3})
assert.NoError(err)
v4 := types.Number(4)
m4 := types.NewStruct("Meta", types.StructData{
"name": types.String("three"),
})
source, err = source.Commit(v4, CommitOptions{Meta: m4})
assert.NoError(err)
h4 := source.Head()
sink, err = sink.Pull(source.Database(), types.NewRef(h2), 1, nil)
assert.NoError(err)
sink, err = sink.Pull(source.Database(), types.NewRef(h4), 1, nil)
assert.NoError(err)
assert.True(source.Head().Equals(sink.Head()))
}
func TestNewCommit(t *testing.T) {
assert := assert.New(t)
commitFieldNames := []string{MetaField, ParentsField, ValueField}
assertTypeEquals := func(e, a *types.Type) {
assert.True(a.Equals(e), "Actual: %s\nExpected %s", a.Describe(), e.Describe())
}
commit := NewCommit(types.Number(1), types.NewSet(), types.EmptyStruct)
at := commit.Type()
et := types.MakeStructType("Commit", commitFieldNames, []*types.Type{
types.EmptyStructType,
types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))),
types.NumberType,
})
assertTypeEquals(et, at)
// Commiting another Number
commit2 := NewCommit(types.Number(2), types.NewSet(types.NewRef(commit)), types.EmptyStruct)
at2 := commit2.Type()
et2 := et
assertTypeEquals(et2, at2)
// Now commit a String
commit3 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2)), types.EmptyStruct)
at3 := commit3.Type()
et3 := types.MakeStructType("Commit", commitFieldNames, []*types.Type{
types.EmptyStructType,
types.MakeSetType(types.MakeRefType(types.MakeStructType("Commit", commitFieldNames, []*types.Type{
types.EmptyStructType,
types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))),
types.MakeUnionType(types.NumberType, types.StringType),
}))),
types.StringType,
})
assertTypeEquals(et3, at3)
// Now commit a String with MetaInfo
meta := types.NewStruct("Meta", types.StructData{"date": types.String("some date"), "number": types.Number(9)})
metaType := types.MakeStructType("Meta", []string{"date", "number"}, []*types.Type{types.StringType, types.NumberType})
assertTypeEquals(metaType, meta.Type())
commit4 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2)), meta)
at4 := commit4.Type()
et4 := types.MakeStructType("Commit", commitFieldNames, []*types.Type{
metaType,
types.MakeSetType(types.MakeRefType(types.MakeStructType("Commit", commitFieldNames, []*types.Type{
types.MakeUnionType(types.EmptyStructType, metaType),
types.MakeSetType(types.MakeRefType(types.MakeCycleType(0))),
types.MakeUnionType(types.NumberType, types.StringType),
}))),
types.StringType,
})
assertTypeEquals(et4, at4)
}
func (suite *LibTestSuite) TestCompositeTypeWithStruct() {
// {"string": "string",
// "list": [false true],
// "struct": {"nested": "string"}
// }
tstruct := types.NewStruct("", types.StructData{
"string": types.String("string"),
"list": types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
"struct": types.NewStruct("", types.StructData{
"nested": types.String("string"),
}),
})
o := NomsValueFromDecodedJSON(map[string]interface{}{
"string": "string",
"list": []interface{}{false, true},
"struct": map[string]interface{}{"nested": "string"},
}, true)
suite.True(tstruct.Equals(o))
}
func metaInfoForCommit(fileOrUrl, source, comment string) types.Struct {
date := time.Now().UTC().Format("2006-01-02T15:04:05-0700")
metaValues := types.StructData{
"date": types.String(date),
fileOrUrl: types.String(source),
}
if comment != "" {
metaValues["comment"] = types.String(comment)
}
return types.NewStruct("Meta", metaValues)
}
func metaInfoForCommit(sourceType, sourceVal, comment string) types.Struct {
date := time.Now().UTC().Format("2006-01-02T15:04:05-0700")
metaValues := map[string]types.Value{
"date": types.String(date),
}
if sourceType != "" {
metaValues[sourceType] = types.String(sourceVal)
}
if comment != "" {
metaValues["comment"] = types.String(comment)
}
return types.NewStruct("Meta", metaValues)
}
// NomsValueFromDecodedJSON takes a generic Go interface{} and recursively
// tries to resolve the types within so that it can build up and return
// a Noms Value with the same structure.
//
// Currently, the only types supported are the Go versions of legal JSON types:
// Primitives:
// - float64
// - bool
// - string
// - nil
//
// Composites:
// - []interface{}
// - map[string]interface{}
func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
switch o := o.(type) {
case string:
return types.String(o)
case bool:
return types.Bool(o)
case float64:
return types.Number(o)
case nil:
return nil
case []interface{}:
items := make([]types.Value, 0, len(o))
for _, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
items = append(items, nv)
}
}
return types.NewList(items...)
case map[string]interface{}:
var v types.Value
if useStruct {
fields := make(map[string]types.Value, len(o))
for k, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
k := types.EscapeStructField(k)
fields[k] = nv
}
}
v = types.NewStruct("", fields)
} else {
kv := make([]types.Value, 0, len(o)*2)
for k, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
kv = append(kv, types.String(k), nv)
}
}
v = types.NewMap(kv...)
}
return v
default:
d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String())
}
return nil
}
func metaInfoForCommit(date, filePath, nomsPath, comment string) types.Struct {
fileOrNomsPath := "inputPath"
path := nomsPath
if path == "" {
path = filePath
fileOrNomsPath = "inputFile"
}
metaValues := types.StructData{
"date": types.String(date),
fileOrNomsPath: types.String(path),
}
if comment != "" {
metaValues["comment"] = types.String(comment)
}
return types.NewStruct("Meta", metaValues)
}
func addPerson(ds dataset.Dataset) {
if flag.NArg() != 4 {
fmt.Fprintln(os.Stderr, "Not enough arguments for command add-person")
return
}
id, err := strconv.ParseUint(flag.Arg(1), 10, 64)
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid person-id: %s", flag.Arg(1))
return
}
np := types.NewStruct("Person", map[string]types.Value{
"id": types.Number(id),
"name": types.String(flag.Arg(2)),
"title": types.String(flag.Arg(3)),
})
ds.Commit(getPersons(ds).Set(types.Number(id), np))
}
func (s *nomsLogTestSuite) TestEmptyCommit() {
str := spec.CreateDatabaseSpecString("ldb", s.LdbDir)
db, err := spec.GetDatabase(str)
s.NoError(err)
ds := dataset.NewDataset(db, "ds1")
meta := types.NewStruct("Meta", map[string]types.Value{
"longNameForTest": types.String("Yoo"),
"test2": types.String("Hoo"),
})
ds, err = ds.Commit(types.String("1"), dataset.CommitOptions{Meta: meta})
s.NoError(err)
ds.Commit(types.String("2"), dataset.CommitOptions{})
db.Close()
dsSpec := spec.CreateValueSpecString("ldb", s.LdbDir, "ds1")
res, _ := s.Run(main, []string{"log", "--show-value=false", dsSpec})
test.EqualsIgnoreHashes(s.T(), metaRes1, res)
res, _ = s.Run(main, []string{"log", "--show-value=false", "--oneline", dsSpec})
test.EqualsIgnoreHashes(s.T(), metaRes2, res)
}
func TestWriteValue(t *testing.T) {
assert := assert.New(t)
factory := chunks.NewMemoryStoreFactory()
defer factory.Shutter()
router = setupWebServer(factory)
defer func() { router = nil }()
testString := "Now, what?"
authKey = "anauthkeyvalue"
w := httptest.NewRecorder()
r, err := newRequest("GET", dbName+constants.RootPath, nil)
assert.NoError(err)
router.ServeHTTP(w, r)
lastRoot := w.Body
assert.Equal(http.StatusOK, w.Code)
craftCommit := func(v types.Value) types.Struct {
return datas.NewCommit(v, types.NewSet(), types.NewStruct("Meta", types.StructData{}))
}
tval := craftCommit(types.Bool(true))
wval := craftCommit(types.String(testString))
chunk1 := types.EncodeValue(tval, nil)
chunk2 := types.EncodeValue(wval, nil)
refMap := types.NewMap(
types.String("ds1"), types.NewRef(tval),
types.String("ds2"), types.NewRef(wval))
chunk3 := types.EncodeValue(refMap, nil)
body := &bytes.Buffer{}
// we would use this func, but it's private so use next line instead: serializeHints(body, map[ref.Ref]struct{}{hint: struct{}{}})
err = binary.Write(body, binary.BigEndian, uint32(0))
assert.NoError(err)
chunks.Serialize(chunk1, body)
chunks.Serialize(chunk2, body)
chunks.Serialize(chunk3, body)
w = httptest.NewRecorder()
r, err = newRequest("POST", dbName+constants.WriteValuePath+"?access_token="+authKey, ioutil.NopCloser(body))
assert.NoError(err)
router.ServeHTTP(w, r)
assert.Equal(http.StatusCreated, w.Code)
w = httptest.NewRecorder()
args := fmt.Sprintf("&last=%s¤t=%s", lastRoot, types.NewRef(refMap).TargetHash())
r, _ = newRequest("POST", dbName+constants.RootPath+"?access_token="+authKey+args, ioutil.NopCloser(body))
router.ServeHTTP(w, r)
assert.Equal(http.StatusOK, w.Code, string(w.Body.Bytes()))
whash := wval.Hash()
hints := map[hash.Hash]struct{}{whash: struct{}{}}
rdr := buildGetRefsRequestBody(hints)
r, _ = newRequest("POST", dbName+constants.GetRefsPath, rdr)
r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, r)
assert.Equal(http.StatusOK, w.Code, string(w.Body.Bytes()))
ms := chunks.NewMemoryStore()
chunks.Deserialize(w.Body, ms, nil)
v := types.DecodeValue(ms.Get(whash), datas.NewDatabase(ms))
assert.Equal(testString, string(v.(types.Struct).Get(datas.ValueField).(types.String)))
}