func TestIncrementalLoadList(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
vs := newLocalValueStore(cs)
expected := NewList(testVals...)
ref := vs.WriteValue(expected).TargetHash()
actualVar := vs.ReadValue(ref)
actual := actualVar.(List)
expectedCount := cs.Reads
assert.Equal(1, expectedCount)
// There will be one read per chunk.
chunkReads := make([]int, expected.Len())
for i := uint64(0); i < expected.Len(); i++ {
v := actual.Get(i)
assert.True(expected.Get(i).Equals(v))
expectedCount += isEncodedOutOfLine(v)
assert.Equal(expectedCount+chunkReads[i], cs.Reads)
// Do it again to make sure multiple derefs don't do multiple loads.
v = actual.Get(i)
assert.Equal(expectedCount+chunkReads[i], cs.Reads)
}
}
func TestHintsOnCache(t *testing.T) {
assert := assert.New(t)
cvs := newLocalValueStore(chunks.NewTestStore())
cr1 := cvs.WriteValue(Number(1))
cr2 := cvs.WriteValue(Number(2))
s1 := NewStruct("", structData{
"a": cr1,
"b": cr2,
})
r := cvs.WriteValue(s1)
v := cvs.ReadValue(r.TargetHash())
if assert.True(v.Equals(s1)) {
cr3 := cvs.WriteValue(Number(3))
s2 := NewStruct("", structData{
"a": cr1,
"b": cr2,
"c": cr3,
})
hints := cvs.chunkHintsFromCache(s2)
if assert.Len(hints, 1) {
for _, hash := range []hash.Hash{r.TargetHash()} {
_, present := hints[hash]
assert.True(present)
}
}
}
}
func SkipTestIncrementalAddRef(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
vs := newLocalValueStore(cs)
expectedItem := Number(42)
ref := vs.WriteValue(expectedItem)
expected := NewList(ref)
ref = vs.WriteValue(expected)
actualVar := vs.ReadValue(ref.TargetHash())
assert.Equal(1, cs.Reads)
assert.True(expected.Equals(actualVar))
actual := actualVar.(List)
actualItem := actual.Get(0)
assert.Equal(2, cs.Reads)
assert.True(expectedItem.Equals(actualItem))
// do it again to make sure caching works.
actualItem = actual.Get(0)
assert.Equal(2, cs.Reads)
assert.True(expectedItem.Equals(actualItem))
}
func TestHandlePostRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
vs := types.NewValueStore(types.NewBatchStoreAdaptor(cs))
commit := NewCommit(types.String("head"), types.NewSet(), types.NewStruct("Meta", types.StructData{}))
newHead := types.NewMap(types.String("dataset1"), vs.WriteValue(commit))
chnx := []chunks.Chunk{
chunks.NewChunk([]byte("abc")),
types.EncodeValue(newHead, nil),
}
err := cs.PutMany(chnx)
assert.NoError(err)
// First attempt should fail, as 'last' won't match.
u := &url.URL{}
queryParams := url.Values{}
queryParams.Add("last", chnx[0].Hash().String())
queryParams.Add("current", chnx[1].Hash().String())
u.RawQuery = queryParams.Encode()
url := u.String()
w := httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
// Now, update the root manually to 'last' and try again.
assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
w = httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func TestHandlePostRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
input1, input2 := "abc", "def"
chnx := []chunks.Chunk{
chunks.NewChunk([]byte(input1)),
chunks.NewChunk([]byte(input2)),
}
err := cs.PutMany(chnx)
assert.NoError(err)
// First attempt should fail, as 'last' won't match.
u := &url.URL{}
queryParams := url.Values{}
queryParams.Add("last", chnx[0].Hash().String())
queryParams.Add("current", chnx[1].Hash().String())
u.RawQuery = queryParams.Encode()
w := httptest.NewRecorder()
HandleRootPost(w, &http.Request{URL: u, Method: "POST"}, params{}, cs)
assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
// Now, update the root manually to 'last' and try again.
assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
w = httptest.NewRecorder()
HandleRootPost(w, &http.Request{URL: u, Method: "POST"}, params{}, cs)
assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func TestHandleGetRefs(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
input1, input2 := "abc", "def"
chnx := []chunks.Chunk{
chunks.NewChunk([]byte(input1)),
chunks.NewChunk([]byte(input2)),
}
err := cs.PutMany(chnx)
assert.NoError(err)
body := strings.NewReader(fmt.Sprintf("ref=%s&ref=%s", chnx[0].Hash(), chnx[1].Hash()))
w := httptest.NewRecorder()
HandleGetRefs(w,
&http.Request{Body: ioutil.NopCloser(body), Method: "POST", Header: http.Header{
"Content-Type": {"application/x-www-form-urlencoded"},
}},
params{},
cs,
)
if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
chunkChan := make(chan *chunks.Chunk)
go chunks.DeserializeToChan(w.Body, chunkChan)
for c := range chunkChan {
assert.Equal(chnx[0].Hash(), c.Hash())
chnx = chnx[1:]
}
assert.Empty(chnx)
}
}
func TestHandleWriteValue(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
ds := NewDatabase(cs)
l := types.NewList(
ds.WriteValue(types.Bool(true)),
ds.WriteValue(types.Bool(false)),
)
ds.WriteValue(l)
hint := l.Hash()
newItem := types.NewEmptyBlob()
itemChunk := types.EncodeValue(newItem, nil)
l2 := l.Insert(1, types.NewRef(newItem))
listChunk := types.EncodeValue(l2, nil)
body := &bytes.Buffer{}
serializeHints(body, map[hash.Hash]struct{}{hint: struct{}{}})
sz := chunks.NewSerializer(body)
sz.Put(itemChunk)
sz.Put(listChunk)
sz.Close()
w := httptest.NewRecorder()
HandleWriteValue(w, &http.Request{Body: ioutil.NopCloser(body), Method: "POST"}, params{}, cs)
if assert.Equal(http.StatusCreated, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
ds2 := NewDatabase(cs)
v := ds2.ReadValue(l2.Hash())
if assert.NotNil(v) {
assert.True(v.Equals(l2), "%+v != %+v", v, l2)
}
}
}
func (suite *RemoteDatabaseSuite) SetupTest() {
suite.cs = chunks.NewTestStore()
suite.makeDs = func(cs chunks.ChunkStore) Database {
hbs := newHTTPBatchStoreForTest(cs)
return &RemoteDatabaseClient{newDatabaseCommon(newCachingChunkHaver(hbs), types.NewValueStore(hbs), hbs)}
}
suite.ds = suite.makeDs(suite.cs)
}
func TestCheckChunksNotInCache(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
cvs := newLocalValueStore(cs)
b := NewEmptyBlob()
cs.Put(EncodeValue(b, nil))
bref := NewRef(b)
assert.Panics(func() { cvs.chunkHintsFromCache(bref) })
}
func TestCheckChunksInCache(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
cvs := newLocalValueStore(cs)
b := NewEmptyBlob()
cs.Put(EncodeValue(b, nil))
cvs.set(b.Hash(), hintedChunk{b.Type(), b.Hash()})
bref := NewRef(b)
assert.NotPanics(func() { cvs.chunkHintsFromCache(bref) })
}
func TestListDiffLargeWithSameMiddle(t *testing.T) {
if testing.Short() {
t.Skip("Skipping test in short mode.")
}
assert := assert.New(t)
cs1 := chunks.NewTestStore()
vs1 := newLocalValueStore(cs1)
nums1 := generateNumbersAsValues(4000)
l1 := NewList(nums1...)
ref1 := vs1.WriteValue(l1).TargetHash()
refList1 := vs1.ReadValue(ref1).(List)
cs2 := chunks.NewTestStore()
vs2 := newLocalValueStore(cs2)
nums2 := generateNumbersAsValuesFromToBy(5, 3550, 1)
l2 := NewList(nums2...)
ref2 := vs2.WriteValue(l2).TargetHash()
refList2 := vs2.ReadValue(ref2).(List)
// diff lists without value store
diff1 := accumulateDiffSplices(l2, l1)
assert.Equal(2, len(diff1))
// diff lists from value stores
diff2 := accumulateDiffSplices(refList2, refList1)
assert.Equal(2, len(diff2))
// diff without and with value store should be same
assert.Equal(diff1, diff2)
// should only read/write a "small & reasonably sized portion of the total"
assert.Equal(3, cs1.Writes)
assert.Equal(3, cs1.Reads)
assert.Equal(3, cs2.Writes)
assert.Equal(3, cs2.Reads)
}
func TestCacheOnReadValue(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
cvs := newLocalValueStore(cs)
b := NewEmptyBlob()
bref := cvs.WriteValue(b)
r := cvs.WriteValue(bref)
cvs2 := newLocalValueStore(cs)
v := cvs2.ReadValue(r.TargetHash())
assert.True(bref.Equals(v))
assert.True(cvs2.isPresent(b.Hash()))
assert.True(cvs2.isPresent(b.Hash()))
}
func TestHandleGetRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
c := chunks.NewChunk([]byte("abc"))
cs.Put(c)
assert.True(cs.UpdateRoot(c.Hash(), hash.Hash{}))
w := httptest.NewRecorder()
HandleRootGet(w, &http.Request{Method: "GET"}, params{}, cs)
if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
root := hash.Parse(string(w.Body.Bytes()))
assert.Equal(c.Hash(), root)
}
}
func TestEnsureChunksInCache(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
cvs := newLocalValueStore(cs)
b := NewEmptyBlob()
s := String("oy")
bref := NewRef(b)
sref := NewRef(s)
l := NewList(bref, sref)
cs.Put(EncodeValue(b, nil))
cs.Put(EncodeValue(s, nil))
cs.Put(EncodeValue(l, nil))
assert.NotPanics(func() { cvs.ensureChunksInCache(bref) })
assert.NotPanics(func() { cvs.ensureChunksInCache(l) })
}
func SkipTestIncrementalLoadSet(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
vs := newLocalValueStore(cs)
expected := NewSet(testVals...)
ref := vs.WriteValue(expected).TargetHash()
actualVar := vs.ReadValue(ref)
actual := actualVar.(Set)
expectedCount := cs.Reads
assert.Equal(1, expectedCount)
actual.Iter(func(v Value) (stop bool) {
expectedCount += isEncodedOutOfLine(v)
assert.Equal(expectedCount, cs.Reads)
return
})
}
func TestEnsureChunksFails(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
cvs := newLocalValueStore(cs)
b := NewEmptyBlob()
bref := NewRef(b)
assert.Panics(func() { cvs.ensureChunksInCache(bref) })
s := String("oy")
cs.Put(EncodeValue(b, nil))
cs.Put(EncodeValue(s, nil))
badRef := constructRef(MakeRefType(MakePrimitiveType(BoolKind)), s.Hash(), 1)
l := NewList(bref, badRef)
cs.Put(EncodeValue(l, nil))
assert.Panics(func() { cvs.ensureChunksInCache(l) })
}
func TestCachingChunkHaver(t *testing.T) {
assert := assert.New(t)
ts := chunks.NewTestStore()
ccs := newCachingChunkHaver(ts)
input := "abc"
c := chunks.NewChunk([]byte(input))
assert.False(ccs.Has(c.Hash()))
assert.Equal(ts.Hases, 1)
assert.False(ccs.Has(c.Hash()))
assert.Equal(ts.Hases, 1)
ts.Put(c)
ccs = newCachingChunkHaver(ts)
assert.True(ccs.Has(c.Hash()))
assert.Equal(ts.Hases, 2)
assert.True(ccs.Has(c.Hash()))
assert.Equal(ts.Hases, 2)
}
func TestRejectPostRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
newHead := types.NewMap(types.String("dataset1"), types.String("Not a Head"))
chunk := types.EncodeValue(newHead, nil)
cs.Put(chunk)
// First attempt should fail, as 'last' won't match.
u := &url.URL{}
queryParams := url.Values{}
queryParams.Add("last", chunks.EmptyChunk.Hash().String())
queryParams.Add("current", chunk.Hash().String())
u.RawQuery = queryParams.Encode()
url := u.String()
w := httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusBadRequest, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func TestHandleHasRefs(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
input1, input2 := "abc", "def"
chnx := []chunks.Chunk{
chunks.NewChunk([]byte(input1)),
chunks.NewChunk([]byte(input2)),
}
err := cs.PutMany(chnx)
assert.NoError(err)
absent := hash.Parse("00000000000000000000000000000002")
body := strings.NewReader(fmt.Sprintf("ref=%s&ref=%s&ref=%s", chnx[0].Hash(), chnx[1].Hash(), absent))
w := httptest.NewRecorder()
HandleHasRefs(
w,
newRequest("POST", "", "", body, http.Header{
"Content-Type": {"application/x-www-form-urlencoded"},
}),
params{},
cs,
)
if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
scanner := bufio.NewScanner(w.Body)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
h := hash.Parse(scanner.Text())
scanner.Scan()
if scanner.Text() == "true" {
assert.Equal(chnx[0].Hash(), h)
chnx = chnx[1:]
} else {
assert.Equal(absent, h)
}
}
assert.Empty(chnx)
}
}
func TestPanicOnReadBadVersion(t *testing.T) {
cvs := newLocalValueStore(&badVersionStore{chunks.NewTestStore()})
assert.Panics(t, func() { cvs.ReadValue(hash.Hash{}) })
}
// NewTestValueStore creates a simple struct that satisfies ValueReadWriter and is backed by a chunks.TestStore.
func NewTestValueStore() *ValueStore {
return newLocalValueStore(chunks.NewTestStore())
}
func (suite *LocalDatabaseSuite) SetupTest() {
suite.cs = chunks.NewTestStore()
suite.makeDs = NewDatabase
suite.ds = suite.makeDs(suite.cs)
}
func (suite *HTTPBatchStoreSuite) SetupTest() {
suite.cs = chunks.NewTestStore()
suite.store = newHTTPBatchStoreForTest(suite.cs)
}
func (suite *LocalToLocalSuite) SetupTest() {
suite.sinkCS = chunks.NewTestStore()
suite.sourceCS = chunks.NewTestStore()
suite.sink = NewDatabase(suite.sinkCS)
suite.source = NewDatabase(suite.sourceCS)
}
func (suite *LocalToRemoteSuite) SetupTest() {
suite.sinkCS = chunks.NewTestStore()
suite.sourceCS = chunks.NewTestStore()
suite.sink = makeRemoteDb(suite.sinkCS)
suite.source = NewDatabase(suite.sourceCS)
}
func (suite *RemoteToRemoteSuite) SetupTest() {
suite.sinkCS = chunks.NewTestStore()
suite.sourceCS = chunks.NewTestStore()
suite.sink = makeRemoteDb(suite.sinkCS)
suite.source = makeRemoteDb(suite.sourceCS)
}
func createTestDataset(name string) Dataset {
return NewDataset(datas.NewDatabase(chunks.NewTestStore()), name)
}
func TestPanicOnWriteBadVersion(t *testing.T) {
cvs := newLocalValueStore(&badVersionStore{chunks.NewTestStore()})
assert.Panics(t, func() { cvs.WriteValue(NewEmptyBlob()) })
}