Ejemplo n.º 1
0
func TestIncrementalLoadList(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	vs := newLocalValueStore(cs)

	expected := NewList(testVals...)
	ref := vs.WriteValue(expected).TargetHash()

	actualVar := vs.ReadValue(ref)
	actual := actualVar.(List)

	expectedCount := cs.Reads
	assert.Equal(1, expectedCount)
	// There will be one read per chunk.
	chunkReads := make([]int, expected.Len())
	for i := uint64(0); i < expected.Len(); i++ {
		v := actual.Get(i)
		assert.True(expected.Get(i).Equals(v))

		expectedCount += isEncodedOutOfLine(v)
		assert.Equal(expectedCount+chunkReads[i], cs.Reads)

		// Do it again to make sure multiple derefs don't do multiple loads.
		v = actual.Get(i)
		assert.Equal(expectedCount+chunkReads[i], cs.Reads)
	}
}
Ejemplo n.º 2
0
func TestHintsOnCache(t *testing.T) {
	assert := assert.New(t)
	cvs := newLocalValueStore(chunks.NewTestStore())

	cr1 := cvs.WriteValue(Number(1))
	cr2 := cvs.WriteValue(Number(2))
	s1 := NewStruct("", structData{
		"a": cr1,
		"b": cr2,
	})
	r := cvs.WriteValue(s1)
	v := cvs.ReadValue(r.TargetHash())

	if assert.True(v.Equals(s1)) {
		cr3 := cvs.WriteValue(Number(3))
		s2 := NewStruct("", structData{
			"a": cr1,
			"b": cr2,
			"c": cr3,
		})

		hints := cvs.chunkHintsFromCache(s2)
		if assert.Len(hints, 1) {
			for _, hash := range []hash.Hash{r.TargetHash()} {
				_, present := hints[hash]
				assert.True(present)
			}
		}
	}
}
Ejemplo n.º 3
0
func SkipTestIncrementalAddRef(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	vs := newLocalValueStore(cs)

	expectedItem := Number(42)
	ref := vs.WriteValue(expectedItem)

	expected := NewList(ref)
	ref = vs.WriteValue(expected)
	actualVar := vs.ReadValue(ref.TargetHash())

	assert.Equal(1, cs.Reads)
	assert.True(expected.Equals(actualVar))

	actual := actualVar.(List)
	actualItem := actual.Get(0)
	assert.Equal(2, cs.Reads)
	assert.True(expectedItem.Equals(actualItem))

	// do it again to make sure caching works.
	actualItem = actual.Get(0)
	assert.Equal(2, cs.Reads)
	assert.True(expectedItem.Equals(actualItem))
}
func TestHandlePostRoot(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	vs := types.NewValueStore(types.NewBatchStoreAdaptor(cs))

	commit := NewCommit(types.String("head"), types.NewSet(), types.NewStruct("Meta", types.StructData{}))
	newHead := types.NewMap(types.String("dataset1"), vs.WriteValue(commit))
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte("abc")),
		types.EncodeValue(newHead, nil),
	}
	err := cs.PutMany(chnx)
	assert.NoError(err)

	// First attempt should fail, as 'last' won't match.
	u := &url.URL{}
	queryParams := url.Values{}
	queryParams.Add("last", chnx[0].Hash().String())
	queryParams.Add("current", chnx[1].Hash().String())
	u.RawQuery = queryParams.Encode()
	url := u.String()

	w := httptest.NewRecorder()
	HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
	assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))

	// Now, update the root manually to 'last' and try again.
	assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
	w = httptest.NewRecorder()
	HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
	assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func TestHandlePostRoot(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	input1, input2 := "abc", "def"
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte(input1)),
		chunks.NewChunk([]byte(input2)),
	}
	err := cs.PutMany(chnx)
	assert.NoError(err)

	// First attempt should fail, as 'last' won't match.
	u := &url.URL{}
	queryParams := url.Values{}
	queryParams.Add("last", chnx[0].Hash().String())
	queryParams.Add("current", chnx[1].Hash().String())
	u.RawQuery = queryParams.Encode()

	w := httptest.NewRecorder()
	HandleRootPost(w, &http.Request{URL: u, Method: "POST"}, params{}, cs)
	assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))

	// Now, update the root manually to 'last' and try again.
	assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
	w = httptest.NewRecorder()
	HandleRootPost(w, &http.Request{URL: u, Method: "POST"}, params{}, cs)
	assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func TestHandleGetRefs(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	input1, input2 := "abc", "def"
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte(input1)),
		chunks.NewChunk([]byte(input2)),
	}
	err := cs.PutMany(chnx)
	assert.NoError(err)

	body := strings.NewReader(fmt.Sprintf("ref=%s&ref=%s", chnx[0].Hash(), chnx[1].Hash()))

	w := httptest.NewRecorder()
	HandleGetRefs(w,
		&http.Request{Body: ioutil.NopCloser(body), Method: "POST", Header: http.Header{
			"Content-Type": {"application/x-www-form-urlencoded"},
		}},
		params{},
		cs,
	)

	if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
		chunkChan := make(chan *chunks.Chunk)
		go chunks.DeserializeToChan(w.Body, chunkChan)
		for c := range chunkChan {
			assert.Equal(chnx[0].Hash(), c.Hash())
			chnx = chnx[1:]
		}
		assert.Empty(chnx)
	}
}
func TestHandleWriteValue(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	ds := NewDatabase(cs)

	l := types.NewList(
		ds.WriteValue(types.Bool(true)),
		ds.WriteValue(types.Bool(false)),
	)
	ds.WriteValue(l)

	hint := l.Hash()
	newItem := types.NewEmptyBlob()
	itemChunk := types.EncodeValue(newItem, nil)
	l2 := l.Insert(1, types.NewRef(newItem))
	listChunk := types.EncodeValue(l2, nil)

	body := &bytes.Buffer{}
	serializeHints(body, map[hash.Hash]struct{}{hint: struct{}{}})
	sz := chunks.NewSerializer(body)
	sz.Put(itemChunk)
	sz.Put(listChunk)
	sz.Close()

	w := httptest.NewRecorder()
	HandleWriteValue(w, &http.Request{Body: ioutil.NopCloser(body), Method: "POST"}, params{}, cs)

	if assert.Equal(http.StatusCreated, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
		ds2 := NewDatabase(cs)
		v := ds2.ReadValue(l2.Hash())
		if assert.NotNil(v) {
			assert.True(v.Equals(l2), "%+v != %+v", v, l2)
		}
	}
}
Ejemplo n.º 8
0
func (suite *RemoteDatabaseSuite) SetupTest() {
	suite.cs = chunks.NewTestStore()
	suite.makeDs = func(cs chunks.ChunkStore) Database {
		hbs := newHTTPBatchStoreForTest(cs)
		return &RemoteDatabaseClient{newDatabaseCommon(newCachingChunkHaver(hbs), types.NewValueStore(hbs), hbs)}
	}
	suite.ds = suite.makeDs(suite.cs)
}
Ejemplo n.º 9
0
func TestCheckChunksNotInCache(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	cvs := newLocalValueStore(cs)

	b := NewEmptyBlob()
	cs.Put(EncodeValue(b, nil))

	bref := NewRef(b)
	assert.Panics(func() { cvs.chunkHintsFromCache(bref) })
}
Ejemplo n.º 10
0
func TestCheckChunksInCache(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	cvs := newLocalValueStore(cs)

	b := NewEmptyBlob()
	cs.Put(EncodeValue(b, nil))
	cvs.set(b.Hash(), hintedChunk{b.Type(), b.Hash()})

	bref := NewRef(b)
	assert.NotPanics(func() { cvs.chunkHintsFromCache(bref) })
}
Ejemplo n.º 11
0
func TestListDiffLargeWithSameMiddle(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test in short mode.")
	}

	assert := assert.New(t)

	cs1 := chunks.NewTestStore()
	vs1 := newLocalValueStore(cs1)
	nums1 := generateNumbersAsValues(4000)
	l1 := NewList(nums1...)
	ref1 := vs1.WriteValue(l1).TargetHash()
	refList1 := vs1.ReadValue(ref1).(List)

	cs2 := chunks.NewTestStore()
	vs2 := newLocalValueStore(cs2)
	nums2 := generateNumbersAsValuesFromToBy(5, 3550, 1)
	l2 := NewList(nums2...)
	ref2 := vs2.WriteValue(l2).TargetHash()
	refList2 := vs2.ReadValue(ref2).(List)

	// diff lists without value store
	diff1 := accumulateDiffSplices(l2, l1)
	assert.Equal(2, len(diff1))

	// diff lists from value stores
	diff2 := accumulateDiffSplices(refList2, refList1)
	assert.Equal(2, len(diff2))

	// diff without and with value store should be same
	assert.Equal(diff1, diff2)

	// should only read/write a "small & reasonably sized portion of the total"
	assert.Equal(3, cs1.Writes)
	assert.Equal(3, cs1.Reads)
	assert.Equal(3, cs2.Writes)
	assert.Equal(3, cs2.Reads)
}
Ejemplo n.º 12
0
func TestCacheOnReadValue(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	cvs := newLocalValueStore(cs)

	b := NewEmptyBlob()
	bref := cvs.WriteValue(b)
	r := cvs.WriteValue(bref)

	cvs2 := newLocalValueStore(cs)
	v := cvs2.ReadValue(r.TargetHash())
	assert.True(bref.Equals(v))
	assert.True(cvs2.isPresent(b.Hash()))
	assert.True(cvs2.isPresent(b.Hash()))
}
func TestHandleGetRoot(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	c := chunks.NewChunk([]byte("abc"))
	cs.Put(c)
	assert.True(cs.UpdateRoot(c.Hash(), hash.Hash{}))

	w := httptest.NewRecorder()
	HandleRootGet(w, &http.Request{Method: "GET"}, params{}, cs)

	if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
		root := hash.Parse(string(w.Body.Bytes()))
		assert.Equal(c.Hash(), root)
	}
}
Ejemplo n.º 14
0
func TestEnsureChunksInCache(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	cvs := newLocalValueStore(cs)

	b := NewEmptyBlob()
	s := String("oy")
	bref := NewRef(b)
	sref := NewRef(s)
	l := NewList(bref, sref)

	cs.Put(EncodeValue(b, nil))
	cs.Put(EncodeValue(s, nil))
	cs.Put(EncodeValue(l, nil))

	assert.NotPanics(func() { cvs.ensureChunksInCache(bref) })
	assert.NotPanics(func() { cvs.ensureChunksInCache(l) })
}
Ejemplo n.º 15
0
func SkipTestIncrementalLoadSet(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	vs := newLocalValueStore(cs)

	expected := NewSet(testVals...)
	ref := vs.WriteValue(expected).TargetHash()

	actualVar := vs.ReadValue(ref)
	actual := actualVar.(Set)

	expectedCount := cs.Reads
	assert.Equal(1, expectedCount)
	actual.Iter(func(v Value) (stop bool) {
		expectedCount += isEncodedOutOfLine(v)
		assert.Equal(expectedCount, cs.Reads)
		return
	})
}
Ejemplo n.º 16
0
func TestEnsureChunksFails(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	cvs := newLocalValueStore(cs)

	b := NewEmptyBlob()
	bref := NewRef(b)
	assert.Panics(func() { cvs.ensureChunksInCache(bref) })

	s := String("oy")
	cs.Put(EncodeValue(b, nil))
	cs.Put(EncodeValue(s, nil))

	badRef := constructRef(MakeRefType(MakePrimitiveType(BoolKind)), s.Hash(), 1)
	l := NewList(bref, badRef)

	cs.Put(EncodeValue(l, nil))
	assert.Panics(func() { cvs.ensureChunksInCache(l) })
}
Ejemplo n.º 17
0
func TestCachingChunkHaver(t *testing.T) {
	assert := assert.New(t)
	ts := chunks.NewTestStore()
	ccs := newCachingChunkHaver(ts)
	input := "abc"

	c := chunks.NewChunk([]byte(input))
	assert.False(ccs.Has(c.Hash()))
	assert.Equal(ts.Hases, 1)
	assert.False(ccs.Has(c.Hash()))
	assert.Equal(ts.Hases, 1)

	ts.Put(c)
	ccs = newCachingChunkHaver(ts)
	assert.True(ccs.Has(c.Hash()))
	assert.Equal(ts.Hases, 2)
	assert.True(ccs.Has(c.Hash()))
	assert.Equal(ts.Hases, 2)
}
func TestRejectPostRoot(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()

	newHead := types.NewMap(types.String("dataset1"), types.String("Not a Head"))
	chunk := types.EncodeValue(newHead, nil)
	cs.Put(chunk)

	// First attempt should fail, as 'last' won't match.
	u := &url.URL{}
	queryParams := url.Values{}
	queryParams.Add("last", chunks.EmptyChunk.Hash().String())
	queryParams.Add("current", chunk.Hash().String())
	u.RawQuery = queryParams.Encode()
	url := u.String()

	w := httptest.NewRecorder()
	HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
	assert.Equal(http.StatusBadRequest, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func TestHandleHasRefs(t *testing.T) {
	assert := assert.New(t)
	cs := chunks.NewTestStore()
	input1, input2 := "abc", "def"
	chnx := []chunks.Chunk{
		chunks.NewChunk([]byte(input1)),
		chunks.NewChunk([]byte(input2)),
	}
	err := cs.PutMany(chnx)
	assert.NoError(err)

	absent := hash.Parse("00000000000000000000000000000002")
	body := strings.NewReader(fmt.Sprintf("ref=%s&ref=%s&ref=%s", chnx[0].Hash(), chnx[1].Hash(), absent))

	w := httptest.NewRecorder()
	HandleHasRefs(
		w,
		newRequest("POST", "", "", body, http.Header{
			"Content-Type": {"application/x-www-form-urlencoded"},
		}),
		params{},
		cs,
	)

	if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
		scanner := bufio.NewScanner(w.Body)
		scanner.Split(bufio.ScanWords)
		for scanner.Scan() {
			h := hash.Parse(scanner.Text())
			scanner.Scan()
			if scanner.Text() == "true" {
				assert.Equal(chnx[0].Hash(), h)
				chnx = chnx[1:]
			} else {
				assert.Equal(absent, h)
			}
		}
		assert.Empty(chnx)
	}
}
Ejemplo n.º 20
0
func TestPanicOnReadBadVersion(t *testing.T) {
	cvs := newLocalValueStore(&badVersionStore{chunks.NewTestStore()})
	assert.Panics(t, func() { cvs.ReadValue(hash.Hash{}) })
}
Ejemplo n.º 21
0
// NewTestValueStore creates a simple struct that satisfies ValueReadWriter and is backed by a chunks.TestStore.
func NewTestValueStore() *ValueStore {
	return newLocalValueStore(chunks.NewTestStore())
}
Ejemplo n.º 22
0
func (suite *LocalDatabaseSuite) SetupTest() {
	suite.cs = chunks.NewTestStore()
	suite.makeDs = NewDatabase
	suite.ds = suite.makeDs(suite.cs)
}
Ejemplo n.º 23
0
func (suite *HTTPBatchStoreSuite) SetupTest() {
	suite.cs = chunks.NewTestStore()
	suite.store = newHTTPBatchStoreForTest(suite.cs)
}
Ejemplo n.º 24
0
func (suite *LocalToLocalSuite) SetupTest() {
	suite.sinkCS = chunks.NewTestStore()
	suite.sourceCS = chunks.NewTestStore()
	suite.sink = NewDatabase(suite.sinkCS)
	suite.source = NewDatabase(suite.sourceCS)
}
Ejemplo n.º 25
0
func (suite *LocalToRemoteSuite) SetupTest() {
	suite.sinkCS = chunks.NewTestStore()
	suite.sourceCS = chunks.NewTestStore()
	suite.sink = makeRemoteDb(suite.sinkCS)
	suite.source = NewDatabase(suite.sourceCS)
}
Ejemplo n.º 26
0
func (suite *RemoteToRemoteSuite) SetupTest() {
	suite.sinkCS = chunks.NewTestStore()
	suite.sourceCS = chunks.NewTestStore()
	suite.sink = makeRemoteDb(suite.sinkCS)
	suite.source = makeRemoteDb(suite.sourceCS)
}
Ejemplo n.º 27
0
func createTestDataset(name string) Dataset {
	return NewDataset(datas.NewDatabase(chunks.NewTestStore()), name)
}
Ejemplo n.º 28
0
func TestPanicOnWriteBadVersion(t *testing.T) {
	cvs := newLocalValueStore(&badVersionStore{chunks.NewTestStore()})
	assert.Panics(t, func() { cvs.WriteValue(NewEmptyBlob()) })
}