コード例 #1
0
ファイル: json_to_noms_test.go プロジェクト: Richardphp/noms
func (suite *LibTestSuite) TestCompositeTypes() {
	// [false true]
	suite.EqualValues(
		types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
		NomsValueFromDecodedJSON([]interface{}{false, true}, false))

	// [[false true]]
	suite.EqualValues(
		types.NewList().Append(
			types.NewList().Append(types.Bool(false)).Append(types.Bool(true))),
		NomsValueFromDecodedJSON([]interface{}{[]interface{}{false, true}}, false))

	// {"string": "string",
	//  "list": [false true],
	//  "map": {"nested": "string"}
	// }
	m := types.NewMap(
		types.String("string"),
		types.String("string"),
		types.String("list"),
		types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
		types.String("map"),
		types.NewMap(
			types.String("nested"),
			types.String("string")))
	o := NomsValueFromDecodedJSON(map[string]interface{}{
		"string": "string",
		"list":   []interface{}{false, true},
		"map":    map[string]interface{}{"nested": "string"},
	}, false)

	suite.True(m.Equals(o))
}
コード例 #2
0
ファイル: walk_test.go プロジェクト: Richardphp/noms
func (suite *WalkAllTestSuite) TestWalkComposites() {
	suite.walkWorker(suite.storeAndRef(types.NewList()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewList(types.Bool(false), types.Number(8))), 4)
	suite.walkWorker(suite.storeAndRef(types.NewSet()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewSet(types.Bool(false), types.Number(8))), 4)
	suite.walkWorker(suite.storeAndRef(types.NewMap()), 2)
	suite.walkWorker(suite.storeAndRef(types.NewMap(types.Number(8), types.Bool(true), types.Number(0), types.Bool(false))), 6)
}
コード例 #3
0
func TestHandlePostRoot(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	vs := types.NewValueStore(types.NewBatchStoreAdaptor(cs))

	commit := NewCommit(types.String("head"), types.NewSet(), types.NewStruct("Meta", types.StructData{}))
	newHead := types.NewMap(types.String("dataset1"), vs.WriteValue(commit))
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte("abc")),
		types.EncodeValue(newHead, nil),
	}
	err := cs.PutMany(chnx)
	assert.NoError(err)

	// First attempt should fail, as 'last' won't match.
	u := &url.URL{}
	queryParams := url.Values{}
	queryParams.Add("last", chnx[0].Hash().String())
	queryParams.Add("current", chnx[1].Hash().String())
	u.RawQuery = queryParams.Encode()
	url := u.String()

	w := httptest.NewRecorder()
	HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
	assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))

	// Now, update the root manually to 'last' and try again.
	assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
	w = httptest.NewRecorder()
	HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
	assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
コード例 #4
0
ファイル: nomsfs.go プロジェクト: Richardphp/noms
func start(dataset string, mount mount) {
	ds, err := spec.GetDataset(dataset)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err)
		return
	}

	hv, ok := ds.MaybeHeadValue()
	if ok {
		if !types.IsSubtype(fsType, hv.Type()) {
			fmt.Fprintf(os.Stderr, "Invalid dataset head: expected type '%s' but found type '%s'\n", fsType.Desc.(types.StructDesc).Name, hv.Type().Desc.(types.StructDesc).Name)
			return
		}
	} else {
		rootAttr := makeAttr(0777) // create the root directory with maximally permissive permissions
		rootDir := types.NewStructWithType(directoryType, types.ValueSlice{types.NewMap()})
		rootInode := types.NewStructWithType(inodeType, types.ValueSlice{rootAttr, rootDir})
		hv = types.NewStructWithType(fsType, types.ValueSlice{rootInode})
	}

	mount(&nomsFS{
		FileSystem: pathfs.NewDefaultFileSystem(),
		ds:         ds,
		head:       hv.(types.Struct),
		mdLock:     &sync.Mutex{},
		nodes:      make(map[hash.Hash]*nNode),
	})
}
コード例 #5
0
ファイル: main.go プロジェクト: willhite/noms-old
func getPersons(ds dataset.Dataset) types.Map {
	hv, ok := ds.MaybeHeadValue()
	if ok {
		return hv.(types.Map)
	} else {
		return types.NewMap()
	}
}
コード例 #6
0
ファイル: main.go プロジェクト: Richardphp/noms
func buildMap(count uint64, createFn createValueFn) types.Collection {
	values := make([]types.Value, count*2)
	for i := uint64(0); i < count*2; i++ {
		values[i] = createFn(i)
	}

	return types.NewMap(values...)
}
コード例 #7
0
ファイル: walk_test.go プロジェクト: Richardphp/noms
func (suite *WalkTestSuite) SkipTestSkipMapKey() {
	wholeMap := types.NewMap(suite.mustSkip, suite.shouldSee, suite.shouldSee, suite.shouldSee)
	reached := suite.skipWorker(wholeMap)
	for _, v := range []types.Value{wholeMap, suite.mustSkip, suite.shouldSee, suite.shouldSeeItem} {
		suite.Contains(reached, v, "Doesn't contain %+v", v)
	}
	suite.Len(reached, 8)
}
コード例 #8
0
func (suite *HTTPBatchStoreSuite) TestUpdateRootWithParams() {
	u := fmt.Sprintf("http://localhost:9000?access_token=%s&other=19", testAuthToken)
	store := newAuthenticatingHTTPBatchStoreForTest(suite, u)
	c := types.EncodeValue(types.NewMap(), nil)
	suite.cs.Put(c)
	suite.True(store.UpdateRoot(c.Hash(), hash.Hash{}))
	suite.Equal(c.Hash(), suite.cs.Root())
}
コード例 #9
0
ファイル: nomsfs.go プロジェクト: Richardphp/noms
func (fs *nomsFS) Mkdir(path string, mode uint32, context *fuse.Context) fuse.Status {
	fs.mdLock.Lock()
	defer fs.mdLock.Unlock()
	_, code := fs.createCommon(path, mode, func() types.Value {
		return types.NewStructWithType(directoryType, types.ValueSlice{types.NewMap()})
	})

	return code
}
コード例 #10
0
ファイル: main.go プロジェクト: Richardphp/noms
func buildMapIncrementally(count uint64, createFn createValueFn) types.Collection {
	m := types.NewMap()

	for i := uint64(0); i < count*2; i += 2 {
		m = m.Set(createFn(i), createFn(i+1))
	}

	return m
}
コード例 #11
0
ファイル: walk_test.go プロジェクト: Richardphp/noms
func (suite *WalkTestSuite) SkipTestSkipMapValue() {
	shouldAlsoSeeItem := types.String("Also good")
	shouldAlsoSee := types.NewSet(shouldAlsoSeeItem)
	wholeMap := types.NewMap(suite.shouldSee, suite.mustSkip, shouldAlsoSee, suite.shouldSee)
	reached := suite.skipWorker(wholeMap)
	for _, v := range []types.Value{wholeMap, suite.shouldSee, suite.shouldSeeItem, suite.mustSkip, shouldAlsoSee, shouldAlsoSeeItem} {
		suite.Contains(reached, v, "Doesn't contain %+v", v)
	}
	suite.Len(reached, 8)
}
コード例 #12
0
ファイル: database_common.go プロジェクト: Richardphp/noms
func (ds *databaseCommon) Datasets() types.Map {
	if ds.datasets == nil {
		if ds.rootRef.IsEmpty() {
			emptyMap := types.NewMap()
			ds.datasets = &emptyMap
		} else {
			ds.datasets = ds.datasetsFromRef(ds.rootRef)
		}
	}

	return *ds.datasets
}
コード例 #13
0
ファイル: pull_test.go プロジェクト: willhite/noms-old
func TestPullTopDown(t *testing.T) {
	assert := assert.New(t)

	sink := createTestDataset("sink")
	source := createTestDataset("source")

	// Give sink and source some initial shared context.
	sourceInitialValue := types.NewMap(
		types.String("first"), NewList(source),
		types.String("second"), NewList(source, types.Number(2)))
	sinkInitialValue := types.NewMap(
		types.String("first"), NewList(sink),
		types.String("second"), NewList(sink, types.Number(2)))

	var err error
	source, err = source.Commit(sourceInitialValue)
	assert.NoError(err)
	sink, err = sink.Commit(sinkInitialValue)
	assert.NoError(err)

	// Add some new stuff to source.
	updatedValue := sourceInitialValue.Set(
		types.String("third"), NewList(source, types.Number(3)))
	source, err = source.Commit(updatedValue)
	assert.NoError(err)

	// Add some more stuff, so that source isn't directly ahead of sink.
	updatedValue = updatedValue.Set(
		types.String("fourth"), NewList(source, types.Number(4)))
	source, err = source.Commit(updatedValue)
	assert.NoError(err)

	sink, err = sink.pull(source.Database(), types.NewRef(source.Head()), 1)
	assert.NoError(err)
	assert.True(source.Head().Equals(sink.Head()))
}
コード例 #14
0
ファイル: util.go プロジェクト: willhite/noms-old
// NomsValueFromDecodedJSON takes a generic Go interface{} and recursively
// tries to resolve the types within so that it can build up and return
// a Noms Value with the same structure.
//
// Currently, the only types supported are the Go versions of legal JSON types:
// Primitives:
//  - float64
//  - bool
//  - string
//  - nil
//
// Composites:
//  - []interface{}
//  - map[string]interface{}
func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
	switch o := o.(type) {
	case string:
		return types.String(o)
	case bool:
		return types.Bool(o)
	case float64:
		return types.Number(o)
	case nil:
		return nil
	case []interface{}:
		items := make([]types.Value, 0, len(o))
		for _, v := range o {
			nv := NomsValueFromDecodedJSON(v, useStruct)
			if nv != nil {
				items = append(items, nv)
			}
		}
		return types.NewList(items...)
	case map[string]interface{}:
		var v types.Value
		if useStruct {
			fields := make(map[string]types.Value, len(o))
			for k, v := range o {
				nv := NomsValueFromDecodedJSON(v, useStruct)
				if nv != nil {
					k := types.EscapeStructField(k)
					fields[k] = nv
				}
			}
			v = types.NewStruct("", fields)
		} else {
			kv := make([]types.Value, 0, len(o)*2)
			for k, v := range o {
				nv := NomsValueFromDecodedJSON(v, useStruct)
				if nv != nil {
					kv = append(kv, types.String(k), nv)
				}
			}
			v = types.NewMap(kv...)
		}
		return v

	default:
		d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String())
	}
	return nil
}
コード例 #15
0
ファイル: read.go プロジェクト: willhite/noms-old
func ReadToMap(r *csv.Reader, headers_raw []string, pkIdx int, kinds KindSlice, vrw types.ValueReadWriter) (m types.Map) {
	headers := make([]string, 0, len(headers_raw)-1)
	for i, h := range headers_raw {
		if i != pkIdx {
			headers = append(headers, types.EscapeStructField(h))
		}
	}

	var pkKind types.NomsKind
	if len(kinds) == 0 {
		pkKind = types.StringKind
	} else {
		pkKind = kinds[pkIdx]
		kinds = append(kinds[:pkIdx], kinds[pkIdx+1:]...)
	}

	t := MakeStructTypeFromHeaders(headers, "", kinds)
	kindMap := make(map[string]types.NomsKind, len(headers))
	t.Desc.(types.StructDesc).IterFields(func(name string, t *types.Type) {
		kindMap[name] = t.Kind()
	})

	m = types.NewMap()
	fields := map[string]types.Value{}
	var pk types.Value
	for {
		row, err := r.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}

		fieldIndex := 0
		for x, v := range row {
			if x == pkIdx {
				pk = StringToType(v, pkKind)
			} else if fieldIndex < len(headers) {
				name := headers[fieldIndex]
				fields[name] = StringToType(v, kindMap[name])
				fieldIndex++
			}
		}
		m = m.Set(pk, types.NewStructWithType(t, fields))
	}
	return
}
コード例 #16
0
ファイル: pull_test.go プロジェクト: willhite/noms-old
func TestPullDeepRefTopDown(t *testing.T) {
	assert := assert.New(t)

	sink := createTestDataset("sink")
	source := createTestDataset("source")

	sourceInitialValue := types.NewList(
		types.NewList(NewList(source)),
		types.NewSet(NewSet(source)),
		types.NewMap(NewMap(source), NewMap(source)))

	source, err := source.Commit(sourceInitialValue)
	assert.NoError(err)

	sink, err = sink.pull(source.Database(), types.NewRef(source.Head()), 1)
	assert.NoError(err)
	assert.True(source.Head().Equals(sink.Head()))
}
コード例 #17
0
func TestRejectPostRoot(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()

	newHead := types.NewMap(types.String("dataset1"), types.String("Not a Head"))
	chunk := types.EncodeValue(newHead, nil)
	cs.Put(chunk)

	// First attempt should fail, as 'last' won't match.
	u := &url.URL{}
	queryParams := url.Values{}
	queryParams.Add("last", chunks.EmptyChunk.Hash().String())
	queryParams.Add("current", chunk.Hash().String())
	u.RawQuery = queryParams.Encode()
	url := u.String()

	w := httptest.NewRecorder()
	HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
	assert.Equal(http.StatusBadRequest, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
コード例 #18
0
ファイル: pull_test.go プロジェクト: willhite/noms-old
func TestPullFirstCommitTopDown(t *testing.T) {
	assert := assert.New(t)

	sink := createTestDataset("sink")
	source := createTestDataset("source")

	sourceInitialValue := types.NewMap(
		types.String("first"), NewList(source),
		types.String("second"), NewList(source, types.Number(2)))

	NewList(sink)
	NewList(sink, types.Number(2))

	source, err := source.Commit(sourceInitialValue)
	assert.NoError(err)

	sink, err = sink.pull(source.Database(), types.NewRef(source.Head()), 1)
	assert.NoError(err)
	assert.True(source.Head().Equals(sink.Head()))
}
コード例 #19
0
ファイル: walk_test.go プロジェクト: Richardphp/noms
func (suite *WalkAllTestSuite) TestWalkNestedComposites() {
	cs := chunks.NewMemoryStore()
	suite.walkWorker(suite.storeAndRef(types.NewList(suite.NewSet(cs), types.Number(8))), 5)
	suite.walkWorker(suite.storeAndRef(types.NewSet(suite.NewList(cs), suite.NewSet(cs))), 6)
	// {"string": "string",
	//  "list": [false true],
	//  "map": {"nested": "string"}
	//  "mtlist": []
	//  "set": [5 7 8]
	//  []: "wow"
	// }
	nested := types.NewMap(
		types.String("string"), types.String("string"),
		types.String("list"), suite.NewList(cs, types.Bool(false), types.Bool(true)),
		types.String("map"), suite.NewMap(cs, types.String("nested"), types.String("string")),
		types.String("mtlist"), suite.NewList(cs),
		types.String("set"), suite.NewSet(cs, types.Number(5), types.Number(7), types.Number(8)),
		suite.NewList(cs), types.String("wow"), // note that the dupe list chunk is skipped
	)
	suite.walkWorker(suite.storeAndRef(nested), 25)
}
コード例 #20
0
ファイル: web_server_test.go プロジェクト: Richardphp/noms
func TestWriteValue(t *testing.T) {
	assert := assert.New(t)
	factory := chunks.NewMemoryStoreFactory()
	defer factory.Shutter()

	router = setupWebServer(factory)
	defer func() { router = nil }()

	testString := "Now, what?"
	authKey = "anauthkeyvalue"

	w := httptest.NewRecorder()
	r, err := newRequest("GET", dbName+constants.RootPath, nil)
	assert.NoError(err)
	router.ServeHTTP(w, r)
	lastRoot := w.Body
	assert.Equal(http.StatusOK, w.Code)

	craftCommit := func(v types.Value) types.Struct {
		return datas.NewCommit(v, types.NewSet(), types.NewStruct("Meta", types.StructData{}))
	}

	tval := craftCommit(types.Bool(true))
	wval := craftCommit(types.String(testString))
	chunk1 := types.EncodeValue(tval, nil)
	chunk2 := types.EncodeValue(wval, nil)
	refMap := types.NewMap(
		types.String("ds1"), types.NewRef(tval),
		types.String("ds2"), types.NewRef(wval))
	chunk3 := types.EncodeValue(refMap, nil)

	body := &bytes.Buffer{}
	// we would use this func, but it's private so use next line instead: serializeHints(body, map[ref.Ref]struct{}{hint: struct{}{}})
	err = binary.Write(body, binary.BigEndian, uint32(0))
	assert.NoError(err)

	chunks.Serialize(chunk1, body)
	chunks.Serialize(chunk2, body)
	chunks.Serialize(chunk3, body)

	w = httptest.NewRecorder()
	r, err = newRequest("POST", dbName+constants.WriteValuePath+"?access_token="+authKey, ioutil.NopCloser(body))
	assert.NoError(err)
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusCreated, w.Code)

	w = httptest.NewRecorder()
	args := fmt.Sprintf("&last=%s&current=%s", lastRoot, types.NewRef(refMap).TargetHash())
	r, _ = newRequest("POST", dbName+constants.RootPath+"?access_token="+authKey+args, ioutil.NopCloser(body))
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusOK, w.Code, string(w.Body.Bytes()))

	whash := wval.Hash()
	hints := map[hash.Hash]struct{}{whash: struct{}{}}
	rdr := buildGetRefsRequestBody(hints)
	r, _ = newRequest("POST", dbName+constants.GetRefsPath, rdr)
	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	router.ServeHTTP(w, r)
	assert.Equal(http.StatusOK, w.Code, string(w.Body.Bytes()))

	ms := chunks.NewMemoryStore()
	chunks.Deserialize(w.Body, ms, nil)
	v := types.DecodeValue(ms.Get(whash), datas.NewDatabase(ms))
	assert.Equal(testString, string(v.(types.Struct).Get(datas.ValueField).(types.String)))
}
コード例 #21
0
func (suite *HTTPBatchStoreSuite) TestUpdateRoot() {
	c := types.EncodeValue(types.NewMap(), nil)
	suite.cs.Put(c)
	suite.True(suite.store.UpdateRoot(c.Hash(), hash.Hash{}))
	suite.Equal(c.Hash(), suite.cs.Root())
}
コード例 #22
0
ファイル: walk_test.go プロジェクト: Richardphp/noms
func (suite *WalkAllTestSuite) NewMap(cs chunks.ChunkStore, vs ...types.Value) types.Ref {
	v := types.NewMap(vs...)
	return suite.vs.WriteValue(v)
}
コード例 #23
0
ファイル: pull_test.go プロジェクト: willhite/noms-old
func NewMap(ds Dataset, vs ...types.Value) types.Ref {
	v := types.NewMap(vs...)
	return ds.Database().WriteValue(v)
}
コード例 #24
0
ファイル: commit.go プロジェクト: willhite/noms-old
func NewMapOfStringToRefOfCommit() types.Map {
	return types.NewMap()
}
コード例 #25
0
ファイル: diff_test.go プロジェクト: Richardphp/noms
func createMap(kv ...interface{}) types.Map {
	keyValues := valsToTypesValues(kv...)
	return types.NewMap(keyValues...)
}
コード例 #26
0
ファイル: xml_importer.go プロジェクト: Richardphp/noms
func main() {
	err := d.Try(func() {
		spec.RegisterDatabaseFlags(flag.CommandLine)
		profile.RegisterProfileFlags(flag.CommandLine)
		flag.Usage = customUsage
		flag.Parse(true)

		if flag.NArg() != 2 {
			d.CheckError(errors.New("Expected directory path followed by dataset"))
		}
		dir := flag.Arg(0)
		ds, err := spec.GetDataset(flag.Arg(1))
		d.CheckError(err)

		defer profile.MaybeStartProfile().Stop()

		cpuCount := runtime.NumCPU()

		filesChan := make(chan fileIndex, 1024)
		refsChan := make(chan refIndex, 1024)

		getFilePaths := func() {
			index := 0
			err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
				d.PanicIfTrue(err != nil, "Cannot traverse directories")
				if !info.IsDir() && filepath.Ext(path) == ".xml" {
					filesChan <- fileIndex{path, index}
					index++
				}

				return nil
			})
			d.PanicIfError(err)
			close(filesChan)
		}

		wg := sync.WaitGroup{}
		importXML := func() {
			expectedType := types.NewMap()
			for f := range filesChan {
				file, err := os.Open(f.path)
				d.PanicIfTrue(err != nil, "Error getting XML")

				xmlObject, err := mxj.NewMapXmlReader(file)
				d.PanicIfTrue(err != nil, "Error decoding XML")
				object := xmlObject.Old()
				file.Close()

				nomsObj := jsontonoms.NomsValueFromDecodedJSON(object, false)
				d.Chk.IsType(expectedType, nomsObj)

				var r types.Ref
				if !*noIO {
					r = ds.Database().WriteValue(nomsObj)
				}

				refsChan <- refIndex{r, f.index}
			}

			wg.Done()
		}

		go getFilePaths()
		for i := 0; i < cpuCount*8; i++ {
			wg.Add(1)
			go importXML()
		}
		go func() {
			wg.Wait()
			close(refsChan) // done converting xml to noms
		}()

		refList := refIndexList{}
		for r := range refsChan {
			refList = append(refList, r)
		}
		sort.Sort(refList)

		refs := make([]types.Value, len(refList))
		for idx, r := range refList {
			refs[idx] = r.ref
		}

		rl := types.NewList(refs...)

		if !*noIO {
			_, err := ds.CommitValue(rl)
			d.PanicIfError(err)
		}
	})

	if err != nil {
		log.Fatal(err)
	}
}
コード例 #27
0
ファイル: nomsfs.go プロジェクト: Richardphp/noms
func makeAttr(mode uint32) types.Struct {
	now := time.Now()
	ctime := types.Number(float64(now.Unix()) + float64(now.Nanosecond())/1000000000)
	mtime := ctime

	user := fuse.CurrentOwner()
	gid := types.Number(float64(user.Gid))
	uid := types.Number(float64(user.Uid))

	return types.NewStructWithType(attrType, types.ValueSlice{ctime, gid, types.Number(mode), mtime, uid, types.NewMap()})
}