func (suite *LibTestSuite) TestCompositeTypes() {
// [false true]
suite.EqualValues(
types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
NomsValueFromDecodedJSON([]interface{}{false, true}, false))
// [[false true]]
suite.EqualValues(
types.NewList().Append(
types.NewList().Append(types.Bool(false)).Append(types.Bool(true))),
NomsValueFromDecodedJSON([]interface{}{[]interface{}{false, true}}, false))
// {"string": "string",
// "list": [false true],
// "map": {"nested": "string"}
// }
m := types.NewMap(
types.String("string"),
types.String("string"),
types.String("list"),
types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
types.String("map"),
types.NewMap(
types.String("nested"),
types.String("string")))
o := NomsValueFromDecodedJSON(map[string]interface{}{
"string": "string",
"list": []interface{}{false, true},
"map": map[string]interface{}{"nested": "string"},
}, false)
suite.True(m.Equals(o))
}
func (suite *WalkAllTestSuite) TestWalkComposites() {
suite.walkWorker(suite.storeAndRef(types.NewList()), 2)
suite.walkWorker(suite.storeAndRef(types.NewList(types.Bool(false), types.Number(8))), 4)
suite.walkWorker(suite.storeAndRef(types.NewSet()), 2)
suite.walkWorker(suite.storeAndRef(types.NewSet(types.Bool(false), types.Number(8))), 4)
suite.walkWorker(suite.storeAndRef(types.NewMap()), 2)
suite.walkWorker(suite.storeAndRef(types.NewMap(types.Number(8), types.Bool(true), types.Number(0), types.Bool(false))), 6)
}
func TestHandlePostRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
vs := types.NewValueStore(types.NewBatchStoreAdaptor(cs))
commit := NewCommit(types.String("head"), types.NewSet(), types.NewStruct("Meta", types.StructData{}))
newHead := types.NewMap(types.String("dataset1"), vs.WriteValue(commit))
chnx := []chunks.Chunk{
chunks.NewChunk([]byte("abc")),
types.EncodeValue(newHead, nil),
}
err := cs.PutMany(chnx)
assert.NoError(err)
// First attempt should fail, as 'last' won't match.
u := &url.URL{}
queryParams := url.Values{}
queryParams.Add("last", chnx[0].Hash().String())
queryParams.Add("current", chnx[1].Hash().String())
u.RawQuery = queryParams.Encode()
url := u.String()
w := httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
// Now, update the root manually to 'last' and try again.
assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
w = httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func start(dataset string, mount mount) {
ds, err := spec.GetDataset(dataset)
if err != nil {
fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err)
return
}
hv, ok := ds.MaybeHeadValue()
if ok {
if !types.IsSubtype(fsType, hv.Type()) {
fmt.Fprintf(os.Stderr, "Invalid dataset head: expected type '%s' but found type '%s'\n", fsType.Desc.(types.StructDesc).Name, hv.Type().Desc.(types.StructDesc).Name)
return
}
} else {
rootAttr := makeAttr(0777) // create the root directory with maximally permissive permissions
rootDir := types.NewStructWithType(directoryType, types.ValueSlice{types.NewMap()})
rootInode := types.NewStructWithType(inodeType, types.ValueSlice{rootAttr, rootDir})
hv = types.NewStructWithType(fsType, types.ValueSlice{rootInode})
}
mount(&nomsFS{
FileSystem: pathfs.NewDefaultFileSystem(),
ds: ds,
head: hv.(types.Struct),
mdLock: &sync.Mutex{},
nodes: make(map[hash.Hash]*nNode),
})
}
func getPersons(ds dataset.Dataset) types.Map {
hv, ok := ds.MaybeHeadValue()
if ok {
return hv.(types.Map)
} else {
return types.NewMap()
}
}
func buildMap(count uint64, createFn createValueFn) types.Collection {
values := make([]types.Value, count*2)
for i := uint64(0); i < count*2; i++ {
values[i] = createFn(i)
}
return types.NewMap(values...)
}
func (suite *WalkTestSuite) SkipTestSkipMapKey() {
wholeMap := types.NewMap(suite.mustSkip, suite.shouldSee, suite.shouldSee, suite.shouldSee)
reached := suite.skipWorker(wholeMap)
for _, v := range []types.Value{wholeMap, suite.mustSkip, suite.shouldSee, suite.shouldSeeItem} {
suite.Contains(reached, v, "Doesn't contain %+v", v)
}
suite.Len(reached, 8)
}
func (suite *HTTPBatchStoreSuite) TestUpdateRootWithParams() {
u := fmt.Sprintf("http://localhost:9000?access_token=%s&other=19", testAuthToken)
store := newAuthenticatingHTTPBatchStoreForTest(suite, u)
c := types.EncodeValue(types.NewMap(), nil)
suite.cs.Put(c)
suite.True(store.UpdateRoot(c.Hash(), hash.Hash{}))
suite.Equal(c.Hash(), suite.cs.Root())
}
func (fs *nomsFS) Mkdir(path string, mode uint32, context *fuse.Context) fuse.Status {
fs.mdLock.Lock()
defer fs.mdLock.Unlock()
_, code := fs.createCommon(path, mode, func() types.Value {
return types.NewStructWithType(directoryType, types.ValueSlice{types.NewMap()})
})
return code
}
func buildMapIncrementally(count uint64, createFn createValueFn) types.Collection {
m := types.NewMap()
for i := uint64(0); i < count*2; i += 2 {
m = m.Set(createFn(i), createFn(i+1))
}
return m
}
func (suite *WalkTestSuite) SkipTestSkipMapValue() {
shouldAlsoSeeItem := types.String("Also good")
shouldAlsoSee := types.NewSet(shouldAlsoSeeItem)
wholeMap := types.NewMap(suite.shouldSee, suite.mustSkip, shouldAlsoSee, suite.shouldSee)
reached := suite.skipWorker(wholeMap)
for _, v := range []types.Value{wholeMap, suite.shouldSee, suite.shouldSeeItem, suite.mustSkip, shouldAlsoSee, shouldAlsoSeeItem} {
suite.Contains(reached, v, "Doesn't contain %+v", v)
}
suite.Len(reached, 8)
}
func (ds *databaseCommon) Datasets() types.Map {
if ds.datasets == nil {
if ds.rootRef.IsEmpty() {
emptyMap := types.NewMap()
ds.datasets = &emptyMap
} else {
ds.datasets = ds.datasetsFromRef(ds.rootRef)
}
}
return *ds.datasets
}
func TestPullTopDown(t *testing.T) {
assert := assert.New(t)
sink := createTestDataset("sink")
source := createTestDataset("source")
// Give sink and source some initial shared context.
sourceInitialValue := types.NewMap(
types.String("first"), NewList(source),
types.String("second"), NewList(source, types.Number(2)))
sinkInitialValue := types.NewMap(
types.String("first"), NewList(sink),
types.String("second"), NewList(sink, types.Number(2)))
var err error
source, err = source.Commit(sourceInitialValue)
assert.NoError(err)
sink, err = sink.Commit(sinkInitialValue)
assert.NoError(err)
// Add some new stuff to source.
updatedValue := sourceInitialValue.Set(
types.String("third"), NewList(source, types.Number(3)))
source, err = source.Commit(updatedValue)
assert.NoError(err)
// Add some more stuff, so that source isn't directly ahead of sink.
updatedValue = updatedValue.Set(
types.String("fourth"), NewList(source, types.Number(4)))
source, err = source.Commit(updatedValue)
assert.NoError(err)
sink, err = sink.pull(source.Database(), types.NewRef(source.Head()), 1)
assert.NoError(err)
assert.True(source.Head().Equals(sink.Head()))
}
// NomsValueFromDecodedJSON takes a generic Go interface{} and recursively
// tries to resolve the types within so that it can build up and return
// a Noms Value with the same structure.
//
// Currently, the only types supported are the Go versions of legal JSON types:
// Primitives:
// - float64
// - bool
// - string
// - nil
//
// Composites:
// - []interface{}
// - map[string]interface{}
func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
switch o := o.(type) {
case string:
return types.String(o)
case bool:
return types.Bool(o)
case float64:
return types.Number(o)
case nil:
return nil
case []interface{}:
items := make([]types.Value, 0, len(o))
for _, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
items = append(items, nv)
}
}
return types.NewList(items...)
case map[string]interface{}:
var v types.Value
if useStruct {
fields := make(map[string]types.Value, len(o))
for k, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
k := types.EscapeStructField(k)
fields[k] = nv
}
}
v = types.NewStruct("", fields)
} else {
kv := make([]types.Value, 0, len(o)*2)
for k, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
kv = append(kv, types.String(k), nv)
}
}
v = types.NewMap(kv...)
}
return v
default:
d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String())
}
return nil
}
func ReadToMap(r *csv.Reader, headers_raw []string, pkIdx int, kinds KindSlice, vrw types.ValueReadWriter) (m types.Map) {
headers := make([]string, 0, len(headers_raw)-1)
for i, h := range headers_raw {
if i != pkIdx {
headers = append(headers, types.EscapeStructField(h))
}
}
var pkKind types.NomsKind
if len(kinds) == 0 {
pkKind = types.StringKind
} else {
pkKind = kinds[pkIdx]
kinds = append(kinds[:pkIdx], kinds[pkIdx+1:]...)
}
t := MakeStructTypeFromHeaders(headers, "", kinds)
kindMap := make(map[string]types.NomsKind, len(headers))
t.Desc.(types.StructDesc).IterFields(func(name string, t *types.Type) {
kindMap[name] = t.Kind()
})
m = types.NewMap()
fields := map[string]types.Value{}
var pk types.Value
for {
row, err := r.Read()
if err == io.EOF {
break
} else if err != nil {
panic(err)
}
fieldIndex := 0
for x, v := range row {
if x == pkIdx {
pk = StringToType(v, pkKind)
} else if fieldIndex < len(headers) {
name := headers[fieldIndex]
fields[name] = StringToType(v, kindMap[name])
fieldIndex++
}
}
m = m.Set(pk, types.NewStructWithType(t, fields))
}
return
}
func TestPullDeepRefTopDown(t *testing.T) {
assert := assert.New(t)
sink := createTestDataset("sink")
source := createTestDataset("source")
sourceInitialValue := types.NewList(
types.NewList(NewList(source)),
types.NewSet(NewSet(source)),
types.NewMap(NewMap(source), NewMap(source)))
source, err := source.Commit(sourceInitialValue)
assert.NoError(err)
sink, err = sink.pull(source.Database(), types.NewRef(source.Head()), 1)
assert.NoError(err)
assert.True(source.Head().Equals(sink.Head()))
}
func TestRejectPostRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
newHead := types.NewMap(types.String("dataset1"), types.String("Not a Head"))
chunk := types.EncodeValue(newHead, nil)
cs.Put(chunk)
// First attempt should fail, as 'last' won't match.
u := &url.URL{}
queryParams := url.Values{}
queryParams.Add("last", chunks.EmptyChunk.Hash().String())
queryParams.Add("current", chunk.Hash().String())
u.RawQuery = queryParams.Encode()
url := u.String()
w := httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusBadRequest, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func TestPullFirstCommitTopDown(t *testing.T) {
assert := assert.New(t)
sink := createTestDataset("sink")
source := createTestDataset("source")
sourceInitialValue := types.NewMap(
types.String("first"), NewList(source),
types.String("second"), NewList(source, types.Number(2)))
NewList(sink)
NewList(sink, types.Number(2))
source, err := source.Commit(sourceInitialValue)
assert.NoError(err)
sink, err = sink.pull(source.Database(), types.NewRef(source.Head()), 1)
assert.NoError(err)
assert.True(source.Head().Equals(sink.Head()))
}
func (suite *WalkAllTestSuite) TestWalkNestedComposites() {
cs := chunks.NewMemoryStore()
suite.walkWorker(suite.storeAndRef(types.NewList(suite.NewSet(cs), types.Number(8))), 5)
suite.walkWorker(suite.storeAndRef(types.NewSet(suite.NewList(cs), suite.NewSet(cs))), 6)
// {"string": "string",
// "list": [false true],
// "map": {"nested": "string"}
// "mtlist": []
// "set": [5 7 8]
// []: "wow"
// }
nested := types.NewMap(
types.String("string"), types.String("string"),
types.String("list"), suite.NewList(cs, types.Bool(false), types.Bool(true)),
types.String("map"), suite.NewMap(cs, types.String("nested"), types.String("string")),
types.String("mtlist"), suite.NewList(cs),
types.String("set"), suite.NewSet(cs, types.Number(5), types.Number(7), types.Number(8)),
suite.NewList(cs), types.String("wow"), // note that the dupe list chunk is skipped
)
suite.walkWorker(suite.storeAndRef(nested), 25)
}
func TestWriteValue(t *testing.T) {
assert := assert.New(t)
factory := chunks.NewMemoryStoreFactory()
defer factory.Shutter()
router = setupWebServer(factory)
defer func() { router = nil }()
testString := "Now, what?"
authKey = "anauthkeyvalue"
w := httptest.NewRecorder()
r, err := newRequest("GET", dbName+constants.RootPath, nil)
assert.NoError(err)
router.ServeHTTP(w, r)
lastRoot := w.Body
assert.Equal(http.StatusOK, w.Code)
craftCommit := func(v types.Value) types.Struct {
return datas.NewCommit(v, types.NewSet(), types.NewStruct("Meta", types.StructData{}))
}
tval := craftCommit(types.Bool(true))
wval := craftCommit(types.String(testString))
chunk1 := types.EncodeValue(tval, nil)
chunk2 := types.EncodeValue(wval, nil)
refMap := types.NewMap(
types.String("ds1"), types.NewRef(tval),
types.String("ds2"), types.NewRef(wval))
chunk3 := types.EncodeValue(refMap, nil)
body := &bytes.Buffer{}
// we would use this func, but it's private so use next line instead: serializeHints(body, map[ref.Ref]struct{}{hint: struct{}{}})
err = binary.Write(body, binary.BigEndian, uint32(0))
assert.NoError(err)
chunks.Serialize(chunk1, body)
chunks.Serialize(chunk2, body)
chunks.Serialize(chunk3, body)
w = httptest.NewRecorder()
r, err = newRequest("POST", dbName+constants.WriteValuePath+"?access_token="+authKey, ioutil.NopCloser(body))
assert.NoError(err)
router.ServeHTTP(w, r)
assert.Equal(http.StatusCreated, w.Code)
w = httptest.NewRecorder()
args := fmt.Sprintf("&last=%s¤t=%s", lastRoot, types.NewRef(refMap).TargetHash())
r, _ = newRequest("POST", dbName+constants.RootPath+"?access_token="+authKey+args, ioutil.NopCloser(body))
router.ServeHTTP(w, r)
assert.Equal(http.StatusOK, w.Code, string(w.Body.Bytes()))
whash := wval.Hash()
hints := map[hash.Hash]struct{}{whash: struct{}{}}
rdr := buildGetRefsRequestBody(hints)
r, _ = newRequest("POST", dbName+constants.GetRefsPath, rdr)
r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, r)
assert.Equal(http.StatusOK, w.Code, string(w.Body.Bytes()))
ms := chunks.NewMemoryStore()
chunks.Deserialize(w.Body, ms, nil)
v := types.DecodeValue(ms.Get(whash), datas.NewDatabase(ms))
assert.Equal(testString, string(v.(types.Struct).Get(datas.ValueField).(types.String)))
}
func (suite *HTTPBatchStoreSuite) TestUpdateRoot() {
c := types.EncodeValue(types.NewMap(), nil)
suite.cs.Put(c)
suite.True(suite.store.UpdateRoot(c.Hash(), hash.Hash{}))
suite.Equal(c.Hash(), suite.cs.Root())
}
func (suite *WalkAllTestSuite) NewMap(cs chunks.ChunkStore, vs ...types.Value) types.Ref {
v := types.NewMap(vs...)
return suite.vs.WriteValue(v)
}
func NewMap(ds Dataset, vs ...types.Value) types.Ref {
v := types.NewMap(vs...)
return ds.Database().WriteValue(v)
}
func NewMapOfStringToRefOfCommit() types.Map {
return types.NewMap()
}
func createMap(kv ...interface{}) types.Map {
keyValues := valsToTypesValues(kv...)
return types.NewMap(keyValues...)
}
func main() {
err := d.Try(func() {
spec.RegisterDatabaseFlags(flag.CommandLine)
profile.RegisterProfileFlags(flag.CommandLine)
flag.Usage = customUsage
flag.Parse(true)
if flag.NArg() != 2 {
d.CheckError(errors.New("Expected directory path followed by dataset"))
}
dir := flag.Arg(0)
ds, err := spec.GetDataset(flag.Arg(1))
d.CheckError(err)
defer profile.MaybeStartProfile().Stop()
cpuCount := runtime.NumCPU()
filesChan := make(chan fileIndex, 1024)
refsChan := make(chan refIndex, 1024)
getFilePaths := func() {
index := 0
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
d.PanicIfTrue(err != nil, "Cannot traverse directories")
if !info.IsDir() && filepath.Ext(path) == ".xml" {
filesChan <- fileIndex{path, index}
index++
}
return nil
})
d.PanicIfError(err)
close(filesChan)
}
wg := sync.WaitGroup{}
importXML := func() {
expectedType := types.NewMap()
for f := range filesChan {
file, err := os.Open(f.path)
d.PanicIfTrue(err != nil, "Error getting XML")
xmlObject, err := mxj.NewMapXmlReader(file)
d.PanicIfTrue(err != nil, "Error decoding XML")
object := xmlObject.Old()
file.Close()
nomsObj := jsontonoms.NomsValueFromDecodedJSON(object, false)
d.Chk.IsType(expectedType, nomsObj)
var r types.Ref
if !*noIO {
r = ds.Database().WriteValue(nomsObj)
}
refsChan <- refIndex{r, f.index}
}
wg.Done()
}
go getFilePaths()
for i := 0; i < cpuCount*8; i++ {
wg.Add(1)
go importXML()
}
go func() {
wg.Wait()
close(refsChan) // done converting xml to noms
}()
refList := refIndexList{}
for r := range refsChan {
refList = append(refList, r)
}
sort.Sort(refList)
refs := make([]types.Value, len(refList))
for idx, r := range refList {
refs[idx] = r.ref
}
rl := types.NewList(refs...)
if !*noIO {
_, err := ds.CommitValue(rl)
d.PanicIfError(err)
}
})
if err != nil {
log.Fatal(err)
}
}
func makeAttr(mode uint32) types.Struct {
now := time.Now()
ctime := types.Number(float64(now.Unix()) + float64(now.Nanosecond())/1000000000)
mtime := ctime
user := fuse.CurrentOwner()
gid := types.Number(float64(user.Gid))
uid := types.Number(float64(user.Uid))
return types.NewStructWithType(attrType, types.ValueSlice{ctime, gid, types.Number(mode), mtime, uid, types.NewMap()})
}
