func TestHandleWriteValue(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
ds := NewDatabase(cs)
l := types.NewList(
ds.WriteValue(types.Bool(true)),
ds.WriteValue(types.Bool(false)),
)
ds.WriteValue(l)
hint := l.Hash()
newItem := types.NewEmptyBlob()
itemChunk := types.EncodeValue(newItem, nil)
l2 := l.Insert(1, types.NewRef(newItem))
listChunk := types.EncodeValue(l2, nil)
body := &bytes.Buffer{}
serializeHints(body, map[hash.Hash]struct{}{hint: struct{}{}})
sz := chunks.NewSerializer(body)
sz.Put(itemChunk)
sz.Put(listChunk)
sz.Close()
w := httptest.NewRecorder()
HandleWriteValue(w, &http.Request{Body: ioutil.NopCloser(body), Method: "POST"}, params{}, cs)
if assert.Equal(http.StatusCreated, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
ds2 := NewDatabase(cs)
v := ds2.ReadValue(l2.Hash())
if assert.NotNil(v) {
assert.True(v.Equals(l2), "%+v != %+v", v, l2)
}
}
}
func TestHandleWriteValueBackpressure(t *testing.T) {
assert := assert.New(t)
cs := &backpressureCS{ChunkStore: chunks.NewMemoryStore()}
ds := NewDatabase(cs)
l := types.NewList(
ds.WriteValue(types.Bool(true)),
ds.WriteValue(types.Bool(false)),
)
ds.WriteValue(l)
hint := l.Hash()
newItem := types.NewEmptyBlob()
itemChunk := types.EncodeValue(newItem, nil)
l2 := l.Insert(1, types.NewRef(newItem))
listChunk := types.EncodeValue(l2, nil)
body := &bytes.Buffer{}
serializeHints(body, map[hash.Hash]struct{}{hint: struct{}{}})
sz := chunks.NewSerializer(body)
sz.Put(itemChunk)
sz.Put(listChunk)
sz.Close()
w := httptest.NewRecorder()
HandleWriteValue(w, &http.Request{Body: ioutil.NopCloser(body), Method: "POST"}, params{}, cs)
if assert.Equal(httpStatusTooManyRequests, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
hashes := deserializeHashes(w.Body)
assert.Len(hashes, 1)
assert.Equal(l2.Hash(), hashes[0])
}
}
func TestWriteValue(t *testing.T) {
assert := assert.New(t)
factory := chunks.NewMemoryStoreFactory()
defer factory.Shutter()
router = setupWebServer(factory)
defer func() { router = nil }()
testString := "Now, what?"
authKey = "anauthkeyvalue"
w := httptest.NewRecorder()
r, err := newRequest("GET", dbName+constants.RootPath, nil)
assert.NoError(err)
router.ServeHTTP(w, r)
lastRoot := w.Body
assert.Equal(http.StatusOK, w.Code)
tval := types.Bool(true)
wval := types.String(testString)
chunk1 := types.EncodeValue(tval, nil)
chunk2 := types.EncodeValue(wval, nil)
refList := types.NewList(types.NewRef(tval), types.NewRef(wval))
chunk3 := types.EncodeValue(refList, nil)
body := &bytes.Buffer{}
// we would use this func, but it's private so use next line instead: serializeHints(body, map[ref.Ref]struct{}{hint: struct{}{}})
err = binary.Write(body, binary.BigEndian, uint32(0))
assert.NoError(err)
chunks.Serialize(chunk1, body)
chunks.Serialize(chunk2, body)
chunks.Serialize(chunk3, body)
w = httptest.NewRecorder()
r, err = newRequest("POST", dbName+constants.WriteValuePath+"?access_token="+authKey, ioutil.NopCloser(body))
assert.NoError(err)
router.ServeHTTP(w, r)
assert.Equal(http.StatusCreated, w.Code)
w = httptest.NewRecorder()
args := fmt.Sprintf("&last=%s¤t=%s", lastRoot, types.NewRef(refList).TargetHash())
r, _ = newRequest("POST", dbName+constants.RootPath+"?access_token="+authKey+args, ioutil.NopCloser(body))
router.ServeHTTP(w, r)
assert.Equal(http.StatusOK, w.Code)
whash := wval.Hash()
hints := map[hash.Hash]struct{}{whash: struct{}{}}
rdr := buildGetRefsRequestBody(hints)
r, _ = newRequest("POST", dbName+constants.GetRefsPath, rdr)
r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, r)
assert.Equal(http.StatusOK, w.Code)
ms := chunks.NewMemoryStore()
chunks.Deserialize(w.Body, ms, nil)
v := types.DecodeValue(ms.Get(whash), datas.NewDatabase(ms))
assert.Equal(testString, string(v.(types.String)))
}
func (suite *HTTPBatchStoreSuite) TestPutChunksInOrder() {
vals := []types.Value{
types.String("abc"),
types.String("def"),
}
l := types.NewList()
for _, val := range vals {
suite.store.SchedulePut(types.EncodeValue(val, nil), 1, types.Hints{})
l = l.Append(types.NewRef(val))
}
suite.store.SchedulePut(types.EncodeValue(l, nil), 2, types.Hints{})
suite.store.Flush()
suite.Equal(3, suite.cs.Writes)
}
func TestHandlePostRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
vs := types.NewValueStore(types.NewBatchStoreAdaptor(cs))
commit := NewCommit(types.String("head"), types.NewSet(), types.NewStruct("Meta", types.StructData{}))
newHead := types.NewMap(types.String("dataset1"), vs.WriteValue(commit))
chnx := []chunks.Chunk{
chunks.NewChunk([]byte("abc")),
types.EncodeValue(newHead, nil),
}
err := cs.PutMany(chnx)
assert.NoError(err)
// First attempt should fail, as 'last' won't match.
u := &url.URL{}
queryParams := url.Values{}
queryParams.Add("last", chnx[0].Hash().String())
queryParams.Add("current", chnx[1].Hash().String())
u.RawQuery = queryParams.Encode()
url := u.String()
w := httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusConflict, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
// Now, update the root manually to 'last' and try again.
assert.True(cs.UpdateRoot(chnx[0].Hash(), hash.Hash{}))
w = httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func (suite *LevelDBPutCacheSuite) TestClearParallel() {
keepIdx := 2
toClear1, toClear2 := hashSet{}, hashSet{}
for i, v := range suite.values {
suite.cache.Insert(types.EncodeValue(v, nil), 1)
if i < keepIdx {
toClear1.Insert(v.Hash())
} else if i > keepIdx {
toClear2.Insert(v.Hash())
}
}
wg := &sync.WaitGroup{}
wg.Add(2)
clear := func(hs hashSet) {
suite.cache.Clear(hs)
wg.Done()
}
go clear(toClear1)
go clear(toClear2)
wg.Wait()
for i, v := range suite.values {
if i == keepIdx {
suite.True(suite.cache.has(v.Hash()))
continue
}
suite.False(suite.cache.has(v.Hash()))
}
}
func (suite *HTTPBatchStoreSuite) TestPutChunk() {
c := types.EncodeValue(types.String("abc"), nil)
suite.store.SchedulePut(c, 1, types.Hints{})
suite.store.Flush()
suite.Equal(1, suite.cs.Writes)
}
func (suite *HTTPBatchStoreSuite) TestUpdateRootWithParams() {
u := fmt.Sprintf("http://localhost:9000?access_token=%s&other=19", testAuthToken)
store := newAuthenticatingHTTPBatchStoreForTest(suite, u)
c := types.EncodeValue(types.NewMap(), nil)
suite.cs.Put(c)
suite.True(store.UpdateRoot(c.Hash(), hash.Hash{}))
suite.Equal(c.Hash(), suite.cs.Root())
}
func (suite *HTTPBatchStoreSuite) TestPutChunksBackpressure() {
bpcs := &backpressureCS{ChunkStore: suite.cs}
bs := newHTTPBatchStoreForTest(bpcs)
defer bs.Close()
defer bpcs.Close()
vals := []types.Value{
types.String("abc"),
types.String("def"),
}
l := types.NewList()
for _, v := range vals {
bs.SchedulePut(types.EncodeValue(v, nil), 1, types.Hints{})
l = l.Append(types.NewRef(v))
}
bs.SchedulePut(types.EncodeValue(l, nil), 2, types.Hints{})
bs.Flush()
suite.Equal(6, suite.cs.Writes)
}
func (suite *HTTPBatchStoreSuite) TestPutChunkWithHints() {
vals := []types.Value{
types.String("abc"),
types.String("def"),
}
chnx := []chunks.Chunk{
types.EncodeValue(vals[0], nil),
types.EncodeValue(vals[1], nil),
}
suite.NoError(suite.cs.PutMany(chnx))
l := types.NewList(types.NewRef(vals[0]), types.NewRef(vals[1]))
suite.store.SchedulePut(types.EncodeValue(l, nil), 2, types.Hints{
chnx[0].Hash(): struct{}{},
chnx[1].Hash(): struct{}{},
})
suite.store.Flush()
suite.Equal(3, suite.cs.Writes)
}
func (suite *LevelDBPutCacheSuite) SetupTest() {
suite.cache = newOrderedChunkCache()
suite.values = []types.Value{
types.String("abc"),
types.String("def"),
types.String("ghi"),
types.String("jkl"),
types.String("mno"),
}
suite.chnx = map[hash.Hash]chunks.Chunk{}
for _, v := range suite.values {
suite.chnx[v.Hash()] = types.EncodeValue(v, nil)
}
}
func TestRejectPostRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
newHead := types.NewMap(types.String("dataset1"), types.String("Not a Head"))
chunk := types.EncodeValue(newHead, nil)
cs.Put(chunk)
// First attempt should fail, as 'last' won't match.
u := &url.URL{}
queryParams := url.Values{}
queryParams.Add("last", chunks.EmptyChunk.Hash().String())
queryParams.Add("current", chunk.Hash().String())
u.RawQuery = queryParams.Encode()
url := u.String()
w := httptest.NewRecorder()
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, cs)
assert.Equal(http.StatusBadRequest, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
}
func (suite *HTTPBatchStoreSuite) TestUpdateRoot() {
c := types.EncodeValue(types.NewMap(), nil)
suite.cs.Put(c)
suite.True(suite.store.UpdateRoot(c.Hash(), hash.Hash{}))
suite.Equal(c.Hash(), suite.cs.Root())
}