func TestBuildWriteValueRequest(t *testing.T) {
	assert := assert.New(t)
	input1, input2 := "abc", "def"
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte(input1)),
		chunks.NewChunk([]byte(input2)),
	}

	hints := map[hash.Hash]struct{}{
		hash.Parse("sha1-0000000000000000000000000000000000000002"): struct{}{},
		hash.Parse("sha1-0000000000000000000000000000000000000003"): struct{}{},
	}
	compressed := buildWriteValueRequest(serializeChunks(chnx, assert), hints)
	gr := snappy.NewReader(compressed)

	count := 0
	for hint := range deserializeHints(gr) {
		count++
		_, present := hints[hint]
		assert.True(present)
	}
	assert.Equal(len(hints), count)

	chunkChan := make(chan *chunks.Chunk, 16)
	go chunks.DeserializeToChan(gr, chunkChan)
	for c := range chunkChan {
		assert.Equal(chnx[0].Hash(), c.Hash())
		chnx = chnx[1:]
	}
	assert.Empty(chnx)
}
func handleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, cs chunks.ChunkStore) {
	d.PanicIfTrue(req.Method != "POST", "Expected post method.")

	params := req.URL.Query()
	tokens := params["last"]
	d.PanicIfTrue(len(tokens) != 1, `Expected "last" query param value`)
	last := hash.Parse(tokens[0])
	tokens = params["current"]
	d.PanicIfTrue(len(tokens) != 1, `Expected "current" query param value`)
	current := hash.Parse(tokens[0])

	// Ensure that proposed new Root is present in cs
	c := cs.Get(current)
	d.PanicIfTrue(c.IsEmpty(), "Can't set Root to a non-present Chunk")

	// Ensure that proposed new Root is a Map and, if it has anything in it, that it's <String, <RefCommit>>
	v := types.DecodeValue(c, nil)
	d.PanicIfTrue(v.Type().Kind() != types.MapKind, "Root of a Database must be a Map")
	m := v.(types.Map)
	if !m.Empty() && !isMapOfStringToRefOfCommit(m) {
		panic(d.Wrap(fmt.Errorf("Root of a Database must be a Map<String, Ref<Commit>>, not %s", m.Type().Describe())))
	}

	if !cs.UpdateRoot(current, last) {
		w.WriteHeader(http.StatusConflict)
		return
	}
}
Esempio n. 3
0
func (suite *ChunkStoreTestSuite) TestChunkStoreRoot() {
	oldRoot := suite.Store.Root()
	suite.True(oldRoot.IsEmpty())

	bogusRoot := hash.Parse("8habda5skfek1265pc5d5l1orptn5dr0")
	newRoot := hash.Parse("8la6qjbh81v85r6q67lqbfrkmpds14lg")

	// Try to update root with bogus oldRoot
	result := suite.Store.UpdateRoot(newRoot, bogusRoot)
	suite.False(result)

	// Now do a valid root update
	result = suite.Store.UpdateRoot(newRoot, oldRoot)
	suite.True(result)
}
func (suite *ChunkStoreTestSuite) TestChunkStoreRoot() {
	oldRoot := suite.Store.Root()
	suite.True(oldRoot.IsEmpty())

	bogusRoot := hash.Parse("sha1-81c870618113ba29b6f2b396ea3a69c6f1d626c5") // sha1("Bogus, Dude")
	newRoot := hash.Parse("sha1-907d14fb3af2b0d4f18c2d46abe8aedce17367bd")   // sha1("Hello, World")

	// Try to update root with bogus oldRoot
	result := suite.Store.UpdateRoot(newRoot, bogusRoot)
	suite.False(result)

	// Now do a valid root update
	result = suite.Store.UpdateRoot(newRoot, oldRoot)
	suite.True(result)
}
Esempio n. 5
0
func (bhcs *httpBatchStore) hasRefs(hashes hashSet, batch chunks.ReadBatch) {
	// POST http://<host>/hasRefs/. Post body: ref=sha1---&ref=sha1---& Response will be text of lines containing "|ref| |bool|".
	u := *bhcs.host
	u.Path = httprouter.CleanPath(bhcs.host.Path + constants.HasRefsPath)

	req := newRequest("POST", bhcs.auth, u.String(), buildHashesRequest(hashes), http.Header{
		"Accept-Encoding": {"x-snappy-framed"},
		"Content-Type":    {"application/x-www-form-urlencoded"},
	})

	res, err := bhcs.httpClient.Do(req)
	d.Chk.NoError(err)
	reader := resBodyReader(res)
	defer closeResponse(reader)

	d.Chk.True(http.StatusOK == res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))

	scanner := bufio.NewScanner(reader)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		h := hash.Parse(scanner.Text())
		d.Chk.True(scanner.Scan())
		if scanner.Text() == "true" {
			for _, outstanding := range batch[h] {
				// This is a little gross, but OutstandingHas.Satisfy() expects a chunk. It ignores it, though, and just sends 'true' over the channel it's holding.
				outstanding.Satisfy(chunks.EmptyChunk)
			}
		} else {
			for _, outstanding := range batch[h] {
				outstanding.Fail()
			}
		}
		delete(batch, h)
	}
}
func handleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, rt chunks.ChunkStore) {
	d.PanicIfTrue(req.Method != "POST", "Expected post method.")

	params := req.URL.Query()
	tokens := params["last"]
	d.PanicIfTrue(len(tokens) != 1, `Expected "last" query param value`)
	last := hash.Parse(tokens[0])
	tokens = params["current"]
	d.PanicIfTrue(len(tokens) != 1, `Expected "current" query param value`)
	current := hash.Parse(tokens[0])

	if !rt.UpdateRoot(current, last) {
		w.WriteHeader(http.StatusConflict)
		return
	}
}
Esempio n. 7
0
func TestHintRoundTrip(t *testing.T) {
	b := &bytes.Buffer{}
	input := map[hash.Hash]struct{}{
		hash.Parse("00000000000000000000000000000000"): struct{}{},
		hash.Parse("00000000000000000000000000000001"): struct{}{},
		hash.Parse("00000000000000000000000000000002"): struct{}{},
		hash.Parse("00000000000000000000000000000003"): struct{}{},
	}
	serializeHints(b, input)
	output := deserializeHints(b)
	assert.Len(t, output, len(input), "Output has different number of elements than input: %v, %v", output, input)
	for h := range output {
		_, present := input[h]
		assert.True(t, present, "%s is in output but not in input", h)
	}
}
Esempio n. 8
0
func (suite *ChunkStoreTestSuite) TestChunkStoreVersion() {
	oldRoot := suite.Store.Root()
	suite.True(oldRoot.IsEmpty())
	newRoot := hash.Parse("11111222223333344444555556666677")
	suite.True(suite.Store.UpdateRoot(newRoot, oldRoot))

	suite.Equal(constants.NomsVersion, suite.Store.Version())
}
Esempio n. 9
0
func (l *internalLevelDBStore) rootByKey(key []byte) hash.Hash {
	val, err := l.db.Get(key, nil)
	if err == errors.ErrNotFound {
		return hash.Hash{}
	}
	d.Chk.NoError(err)

	return hash.Parse(string(val))
}
Esempio n. 10
0
func (bhcs *httpBatchStore) Root() hash.Hash {
	// GET http://<host>/root. Response will be ref of root.
	res := bhcs.requestRoot("GET", hash.Hash{}, hash.Hash{})
	defer closeResponse(res.Body)

	d.Chk.True(http.StatusOK == res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))
	data, err := ioutil.ReadAll(res.Body)
	d.Chk.NoError(err)
	return hash.Parse(string(data))
}
Esempio n. 11
0
func TestWriteRef(t *testing.T) {
	typ := MakeRefType(NumberType)
	r := hash.Parse("0123456789abcdefghijklmnopqrstuv")

	assertEncoding(t,
		[]interface{}{
			uint8(RefKind), uint8(NumberKind), r.String(), uint64(4),
		},
		constructRef(typ, r, 4),
	)
}
Esempio n. 12
0
func TestWriteCompoundBlob(t *testing.T) {
	r1 := hash.Parse("00000000000000000000000000000001")
	r2 := hash.Parse("00000000000000000000000000000002")
	r3 := hash.Parse("00000000000000000000000000000003")

	assertEncoding(t,
		[]interface{}{
			uint8(BlobKind), true,
			uint32(3), // len
			uint8(RefKind), uint8(BlobKind), r1.String(), uint64(11), uint8(NumberKind), Number(20), uint64(20),
			uint8(RefKind), uint8(BlobKind), r2.String(), uint64(22), uint8(NumberKind), Number(40), uint64(40),
			uint8(RefKind), uint8(BlobKind), r3.String(), uint64(33), uint8(NumberKind), Number(60), uint64(60),
		},
		newBlob(newBlobMetaSequence([]metaTuple{
			newMetaTuple(constructRef(MakeRefType(BlobType), r1, 11), orderedKeyFromInt(20), 20, nil),
			newMetaTuple(constructRef(MakeRefType(BlobType), r2, 22), orderedKeyFromInt(40), 40, nil),
			newMetaTuple(constructRef(MakeRefType(BlobType), r3, 33), orderedKeyFromInt(60), 60, nil),
		}, NewTestValueStore())),
	)
}
func extractHashes(req *http.Request) hash.HashSlice {
	err := req.ParseForm()
	d.PanicIfError(err)
	hashStrs := req.PostForm["ref"]
	d.PanicIfTrue(len(hashStrs) <= 0, "PostForm is empty")

	hashes := make(hash.HashSlice, len(hashStrs))
	for idx, refStr := range hashStrs {
		hashes[idx] = hash.Parse(refStr)
	}
	return hashes
}
func extractHashes(req *http.Request) hash.HashSlice {
	err := req.ParseForm()
	d.Exp.NoError(err)
	hashStrs := req.PostForm["ref"]
	d.Exp.True(len(hashStrs) > 0)

	hashes := make(hash.HashSlice, len(hashStrs))
	for idx, refStr := range hashStrs {
		hashes[idx] = hash.Parse(refStr)
	}
	return hashes
}
func TestBuildHashesRequest(t *testing.T) {
	assert := assert.New(t)
	hashes := map[hash.Hash]struct{}{
		hash.Parse("sha1-0000000000000000000000000000000000000002"): struct{}{},
		hash.Parse("sha1-0000000000000000000000000000000000000003"): struct{}{},
	}
	r := buildHashesRequest(hashes)
	b, err := ioutil.ReadAll(r)
	assert.NoError(err)

	urlValues, err := url.ParseQuery(string(b))
	assert.NoError(err)
	assert.NotEmpty(urlValues)

	queryRefs := urlValues["ref"]
	assert.Len(queryRefs, len(hashes))
	for _, r := range queryRefs {
		_, present := hashes[hash.Parse(r)]
		assert.True(present, "Query contains %s, which is not in initial refs", r)
	}
}
func TestHandleHasRefs(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	input1, input2 := "abc", "def"
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte(input1)),
		chunks.NewChunk([]byte(input2)),
	}
	err := cs.PutMany(chnx)
	assert.NoError(err)

	absent := hash.Parse("00000000000000000000000000000002")
	body := strings.NewReader(fmt.Sprintf("ref=%s&ref=%s&ref=%s", chnx[0].Hash(), chnx[1].Hash(), absent))

	w := httptest.NewRecorder()
	HandleHasRefs(
		w,
		newRequest("POST", "", "", body, http.Header{
			"Content-Type": {"application/x-www-form-urlencoded"},
		}),
		params{},
		cs,
	)

	if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
		scanner := bufio.NewScanner(w.Body)
		scanner.Split(bufio.ScanWords)
		for scanner.Scan() {
			h := hash.Parse(scanner.Text())
			scanner.Scan()
			if scanner.Text() == "true" {
				assert.Equal(chnx[0].Hash(), h)
				chnx = chnx[1:]
			} else {
				assert.Equal(absent, h)
			}
		}
		assert.Empty(chnx)
	}
}
func HandleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, rt chunks.ChunkStore) {
	err := d.Try(func() {
		d.Exp.Equal("POST", req.Method)

		params := req.URL.Query()
		tokens := params["last"]
		d.Exp.Len(tokens, 1)
		last := hash.Parse(tokens[0])
		tokens = params["current"]
		d.Exp.Len(tokens, 1)
		current := hash.Parse(tokens[0])

		if !rt.UpdateRoot(current, last) {
			w.WriteHeader(http.StatusConflict)
			return
		}
	})

	if err != nil {
		http.Error(w, fmt.Sprintf("Error: %v", err), http.StatusBadRequest)
		return
	}
}
func TestHandleGetRoot(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	c := chunks.NewChunk([]byte("abc"))
	cs.Put(c)
	assert.True(cs.UpdateRoot(c.Hash(), hash.Hash{}))

	w := httptest.NewRecorder()
	HandleRootGet(w, &http.Request{Method: "GET"}, params{}, cs)

	if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
		root := hash.Parse(string(w.Body.Bytes()))
		assert.Equal(c.Hash(), root)
	}
}
Esempio n. 19
0
func TestRefSpec(t *testing.T) {
	assert := assert.New(t)

	testCases := []map[string]string{
		map[string]string{"spec": "http://local.attic.io/john/doe::sha1-0123456789012345678901234567890123456789", "scheme": "http", "path": "//local.attic.io/john/doe", "ref": "sha1-0123456789012345678901234567890123456789"},
		map[string]string{"spec": "ldb:/filesys/john/doe::sha1-0123456789012345678901234567890123456789", "scheme": "ldb", "path": "/filesys/john/doe", "ref": "sha1-0123456789012345678901234567890123456789"},
		map[string]string{"spec": "mem::sha1-0123456789012345678901234567890123456789", "scheme": "mem", "ref": "sha1-0123456789012345678901234567890123456789"},
	}

	for _, tc := range testCases {
		refSpec, err := ParseRefSpec(tc["spec"])
		assert.NoError(err)
		dbSpec1 := DatabaseSpec{Protocol: tc["scheme"], Path: tc["path"], accessToken: tc["accessToken"]}
		assert.Equal(RefSpec{DbSpec: dbSpec1, Ref: hash.Parse(tc["ref"])}, refSpec)
	}
}
Esempio n. 20
0
func TestPathSpec(t *testing.T) {
	assert := assert.New(t)

	testCases := []map[string]string{
		map[string]string{"spec": "http://local.attic.io/john/doe::sha1-0123456789012345678901234567890123456789", "scheme": "http", "path": "//local.attic.io/john/doe", "ref": "sha1-0123456789012345678901234567890123456789"},
		map[string]string{"spec": "http://localhost:8000/john/doe/::ds1", "scheme": "http", "path": "//localhost:8000/john/doe/", "ds": "ds1"},
	}

	for _, tc := range testCases {
		pathSpec, err := ParsePathSpec(tc["spec"])
		assert.NoError(err)
		dbSpec1 := DatabaseSpec{Protocol: tc["scheme"], Path: tc["path"], accessToken: tc["accessToken"]}
		if tc["ref"] != "" {
			assert.Equal(&RefSpec{DbSpec: dbSpec1, Ref: hash.Parse(tc["ref"])}, pathSpec.(*RefSpec))
		} else {
			assert.Equal(&DatasetSpec{DbSpec: dbSpec1, DatasetName: tc["ds"]}, pathSpec.(*DatasetSpec))
		}
	}

	_, err := ParsePathSpec("http://local.attic.io")
	assert.Error(err)

}
Esempio n. 21
0
func (r *nomsTestReader) readHash() hash.Hash {
	return hash.Parse(r.readString())
}
Esempio n. 22
0
func TestGetRequestBatch(t *testing.T) {
	assert := assert.New(t)
	r0 := hash.Parse("00000000000000000000000000000000")
	c1 := NewChunk([]byte("abc"))
	r1 := c1.Hash()
	c2 := NewChunk([]byte("123"))
	r2 := c2.Hash()

	tally := func(b bool, trueCnt, falseCnt *int) {
		if b {
			*trueCnt++
		} else {
			*falseCnt++
		}
	}

	req0chan := make(chan bool, 1)
	req1chan := make(chan Chunk, 1)
	req2chan := make(chan bool, 1)
	req3chan := make(chan bool, 1)
	req4chan := make(chan Chunk, 1)

	batch := ReadBatch{
		r0: []OutstandingRequest{OutstandingHas(req0chan), OutstandingGet(req1chan)},
		r1: []OutstandingRequest{OutstandingHas(req2chan)},
		r2: []OutstandingRequest{OutstandingHas(req3chan), OutstandingGet(req4chan)},
	}
	go func() {
		for requestedRef, reqs := range batch {
			for _, req := range reqs {
				if requestedRef == r1 {
					req.Satisfy(c1)
					delete(batch, r1)
				} else if requestedRef == r2 {
					req.Satisfy(c2)
					delete(batch, r2)
				}
			}
		}
	}()
	var r1True, r1False, r2True, r2False int
	for b := range req2chan {
		tally(b, &r1True, &r1False)
	}
	for b := range req3chan {
		tally(b, &r2True, &r2False)
	}
	for c := range req4chan {
		assert.EqualValues(c2.Hash(), c.Hash())
	}

	assert.Equal(1, r1True)
	assert.Equal(0, r1False)
	assert.Equal(1, r2True)
	assert.Equal(0, r2False)

	go batch.Close()
	var r0True, r0False int
	for b := range req0chan {
		tally(b, &r0True, &r0False)
	}
	for c := range req1chan {
		assert.EqualValues(EmptyChunk.Hash(), c.Hash())
	}
	assert.Equal(0, r0True)
	assert.Equal(1, r0False)
}
Esempio n. 23
0
func (suite *ChunkStoreTestSuite) TestChunkStoreGetNonExisting() {
	h := hash.Parse("11111111111111111111111111111111")
	c := suite.Store.Get(h)
	suite.True(c.IsEmpty())
}