func TestBuildWriteValueRequest(t *testing.T) {
assert := assert.New(t)
input1, input2 := "abc", "def"
chnx := []chunks.Chunk{
chunks.NewChunk([]byte(input1)),
chunks.NewChunk([]byte(input2)),
}
hints := map[hash.Hash]struct{}{
hash.Parse("sha1-0000000000000000000000000000000000000002"): struct{}{},
hash.Parse("sha1-0000000000000000000000000000000000000003"): struct{}{},
}
compressed := buildWriteValueRequest(serializeChunks(chnx, assert), hints)
gr := snappy.NewReader(compressed)
count := 0
for hint := range deserializeHints(gr) {
count++
_, present := hints[hint]
assert.True(present)
}
assert.Equal(len(hints), count)
chunkChan := make(chan *chunks.Chunk, 16)
go chunks.DeserializeToChan(gr, chunkChan)
for c := range chunkChan {
assert.Equal(chnx[0].Hash(), c.Hash())
chnx = chnx[1:]
}
assert.Empty(chnx)
}
func handleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, cs chunks.ChunkStore) {
d.PanicIfTrue(req.Method != "POST", "Expected post method.")
params := req.URL.Query()
tokens := params["last"]
d.PanicIfTrue(len(tokens) != 1, `Expected "last" query param value`)
last := hash.Parse(tokens[0])
tokens = params["current"]
d.PanicIfTrue(len(tokens) != 1, `Expected "current" query param value`)
current := hash.Parse(tokens[0])
// Ensure that proposed new Root is present in cs
c := cs.Get(current)
d.PanicIfTrue(c.IsEmpty(), "Can't set Root to a non-present Chunk")
// Ensure that proposed new Root is a Map and, if it has anything in it, that it's <String, <RefCommit>>
v := types.DecodeValue(c, nil)
d.PanicIfTrue(v.Type().Kind() != types.MapKind, "Root of a Database must be a Map")
m := v.(types.Map)
if !m.Empty() && !isMapOfStringToRefOfCommit(m) {
panic(d.Wrap(fmt.Errorf("Root of a Database must be a Map<String, Ref<Commit>>, not %s", m.Type().Describe())))
}
if !cs.UpdateRoot(current, last) {
w.WriteHeader(http.StatusConflict)
return
}
}
func (suite *ChunkStoreTestSuite) TestChunkStoreRoot() {
oldRoot := suite.Store.Root()
suite.True(oldRoot.IsEmpty())
bogusRoot := hash.Parse("8habda5skfek1265pc5d5l1orptn5dr0")
newRoot := hash.Parse("8la6qjbh81v85r6q67lqbfrkmpds14lg")
// Try to update root with bogus oldRoot
result := suite.Store.UpdateRoot(newRoot, bogusRoot)
suite.False(result)
// Now do a valid root update
result = suite.Store.UpdateRoot(newRoot, oldRoot)
suite.True(result)
}
func (suite *ChunkStoreTestSuite) TestChunkStoreRoot() {
oldRoot := suite.Store.Root()
suite.True(oldRoot.IsEmpty())
bogusRoot := hash.Parse("sha1-81c870618113ba29b6f2b396ea3a69c6f1d626c5") // sha1("Bogus, Dude")
newRoot := hash.Parse("sha1-907d14fb3af2b0d4f18c2d46abe8aedce17367bd") // sha1("Hello, World")
// Try to update root with bogus oldRoot
result := suite.Store.UpdateRoot(newRoot, bogusRoot)
suite.False(result)
// Now do a valid root update
result = suite.Store.UpdateRoot(newRoot, oldRoot)
suite.True(result)
}
func (bhcs *httpBatchStore) hasRefs(hashes hashSet, batch chunks.ReadBatch) {
// POST http://<host>/hasRefs/. Post body: ref=sha1---&ref=sha1---& Response will be text of lines containing "|ref| |bool|".
u := *bhcs.host
u.Path = httprouter.CleanPath(bhcs.host.Path + constants.HasRefsPath)
req := newRequest("POST", bhcs.auth, u.String(), buildHashesRequest(hashes), http.Header{
"Accept-Encoding": {"x-snappy-framed"},
"Content-Type": {"application/x-www-form-urlencoded"},
})
res, err := bhcs.httpClient.Do(req)
d.Chk.NoError(err)
reader := resBodyReader(res)
defer closeResponse(reader)
d.Chk.True(http.StatusOK == res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))
scanner := bufio.NewScanner(reader)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
h := hash.Parse(scanner.Text())
d.Chk.True(scanner.Scan())
if scanner.Text() == "true" {
for _, outstanding := range batch[h] {
// This is a little gross, but OutstandingHas.Satisfy() expects a chunk. It ignores it, though, and just sends 'true' over the channel it's holding.
outstanding.Satisfy(chunks.EmptyChunk)
}
} else {
for _, outstanding := range batch[h] {
outstanding.Fail()
}
}
delete(batch, h)
}
}
func handleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, rt chunks.ChunkStore) {
d.PanicIfTrue(req.Method != "POST", "Expected post method.")
params := req.URL.Query()
tokens := params["last"]
d.PanicIfTrue(len(tokens) != 1, `Expected "last" query param value`)
last := hash.Parse(tokens[0])
tokens = params["current"]
d.PanicIfTrue(len(tokens) != 1, `Expected "current" query param value`)
current := hash.Parse(tokens[0])
if !rt.UpdateRoot(current, last) {
w.WriteHeader(http.StatusConflict)
return
}
}
func TestHintRoundTrip(t *testing.T) {
b := &bytes.Buffer{}
input := map[hash.Hash]struct{}{
hash.Parse("00000000000000000000000000000000"): struct{}{},
hash.Parse("00000000000000000000000000000001"): struct{}{},
hash.Parse("00000000000000000000000000000002"): struct{}{},
hash.Parse("00000000000000000000000000000003"): struct{}{},
}
serializeHints(b, input)
output := deserializeHints(b)
assert.Len(t, output, len(input), "Output has different number of elements than input: %v, %v", output, input)
for h := range output {
_, present := input[h]
assert.True(t, present, "%s is in output but not in input", h)
}
}
func (suite *ChunkStoreTestSuite) TestChunkStoreVersion() {
oldRoot := suite.Store.Root()
suite.True(oldRoot.IsEmpty())
newRoot := hash.Parse("11111222223333344444555556666677")
suite.True(suite.Store.UpdateRoot(newRoot, oldRoot))
suite.Equal(constants.NomsVersion, suite.Store.Version())
}
func (l *internalLevelDBStore) rootByKey(key []byte) hash.Hash {
val, err := l.db.Get(key, nil)
if err == errors.ErrNotFound {
return hash.Hash{}
}
d.Chk.NoError(err)
return hash.Parse(string(val))
}
func (bhcs *httpBatchStore) Root() hash.Hash {
// GET http://<host>/root. Response will be ref of root.
res := bhcs.requestRoot("GET", hash.Hash{}, hash.Hash{})
defer closeResponse(res.Body)
d.Chk.True(http.StatusOK == res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))
data, err := ioutil.ReadAll(res.Body)
d.Chk.NoError(err)
return hash.Parse(string(data))
}
func TestWriteRef(t *testing.T) {
typ := MakeRefType(NumberType)
r := hash.Parse("0123456789abcdefghijklmnopqrstuv")
assertEncoding(t,
[]interface{}{
uint8(RefKind), uint8(NumberKind), r.String(), uint64(4),
},
constructRef(typ, r, 4),
)
}
func TestWriteCompoundBlob(t *testing.T) {
r1 := hash.Parse("00000000000000000000000000000001")
r2 := hash.Parse("00000000000000000000000000000002")
r3 := hash.Parse("00000000000000000000000000000003")
assertEncoding(t,
[]interface{}{
uint8(BlobKind), true,
uint32(3), // len
uint8(RefKind), uint8(BlobKind), r1.String(), uint64(11), uint8(NumberKind), Number(20), uint64(20),
uint8(RefKind), uint8(BlobKind), r2.String(), uint64(22), uint8(NumberKind), Number(40), uint64(40),
uint8(RefKind), uint8(BlobKind), r3.String(), uint64(33), uint8(NumberKind), Number(60), uint64(60),
},
newBlob(newBlobMetaSequence([]metaTuple{
newMetaTuple(constructRef(MakeRefType(BlobType), r1, 11), orderedKeyFromInt(20), 20, nil),
newMetaTuple(constructRef(MakeRefType(BlobType), r2, 22), orderedKeyFromInt(40), 40, nil),
newMetaTuple(constructRef(MakeRefType(BlobType), r3, 33), orderedKeyFromInt(60), 60, nil),
}, NewTestValueStore())),
)
}
func extractHashes(req *http.Request) hash.HashSlice {
err := req.ParseForm()
d.PanicIfError(err)
hashStrs := req.PostForm["ref"]
d.PanicIfTrue(len(hashStrs) <= 0, "PostForm is empty")
hashes := make(hash.HashSlice, len(hashStrs))
for idx, refStr := range hashStrs {
hashes[idx] = hash.Parse(refStr)
}
return hashes
}
func extractHashes(req *http.Request) hash.HashSlice {
err := req.ParseForm()
d.Exp.NoError(err)
hashStrs := req.PostForm["ref"]
d.Exp.True(len(hashStrs) > 0)
hashes := make(hash.HashSlice, len(hashStrs))
for idx, refStr := range hashStrs {
hashes[idx] = hash.Parse(refStr)
}
return hashes
}
func TestBuildHashesRequest(t *testing.T) {
assert := assert.New(t)
hashes := map[hash.Hash]struct{}{
hash.Parse("sha1-0000000000000000000000000000000000000002"): struct{}{},
hash.Parse("sha1-0000000000000000000000000000000000000003"): struct{}{},
}
r := buildHashesRequest(hashes)
b, err := ioutil.ReadAll(r)
assert.NoError(err)
urlValues, err := url.ParseQuery(string(b))
assert.NoError(err)
assert.NotEmpty(urlValues)
queryRefs := urlValues["ref"]
assert.Len(queryRefs, len(hashes))
for _, r := range queryRefs {
_, present := hashes[hash.Parse(r)]
assert.True(present, "Query contains %s, which is not in initial refs", r)
}
}
func TestHandleHasRefs(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
input1, input2 := "abc", "def"
chnx := []chunks.Chunk{
chunks.NewChunk([]byte(input1)),
chunks.NewChunk([]byte(input2)),
}
err := cs.PutMany(chnx)
assert.NoError(err)
absent := hash.Parse("00000000000000000000000000000002")
body := strings.NewReader(fmt.Sprintf("ref=%s&ref=%s&ref=%s", chnx[0].Hash(), chnx[1].Hash(), absent))
w := httptest.NewRecorder()
HandleHasRefs(
w,
newRequest("POST", "", "", body, http.Header{
"Content-Type": {"application/x-www-form-urlencoded"},
}),
params{},
cs,
)
if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
scanner := bufio.NewScanner(w.Body)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
h := hash.Parse(scanner.Text())
scanner.Scan()
if scanner.Text() == "true" {
assert.Equal(chnx[0].Hash(), h)
chnx = chnx[1:]
} else {
assert.Equal(absent, h)
}
}
assert.Empty(chnx)
}
}
func HandleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, rt chunks.ChunkStore) {
err := d.Try(func() {
d.Exp.Equal("POST", req.Method)
params := req.URL.Query()
tokens := params["last"]
d.Exp.Len(tokens, 1)
last := hash.Parse(tokens[0])
tokens = params["current"]
d.Exp.Len(tokens, 1)
current := hash.Parse(tokens[0])
if !rt.UpdateRoot(current, last) {
w.WriteHeader(http.StatusConflict)
return
}
})
if err != nil {
http.Error(w, fmt.Sprintf("Error: %v", err), http.StatusBadRequest)
return
}
}
func TestHandleGetRoot(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
c := chunks.NewChunk([]byte("abc"))
cs.Put(c)
assert.True(cs.UpdateRoot(c.Hash(), hash.Hash{}))
w := httptest.NewRecorder()
HandleRootGet(w, &http.Request{Method: "GET"}, params{}, cs)
if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
root := hash.Parse(string(w.Body.Bytes()))
assert.Equal(c.Hash(), root)
}
}
func TestRefSpec(t *testing.T) {
assert := assert.New(t)
testCases := []map[string]string{
map[string]string{"spec": "http://local.attic.io/john/doe::sha1-0123456789012345678901234567890123456789", "scheme": "http", "path": "//local.attic.io/john/doe", "ref": "sha1-0123456789012345678901234567890123456789"},
map[string]string{"spec": "ldb:/filesys/john/doe::sha1-0123456789012345678901234567890123456789", "scheme": "ldb", "path": "/filesys/john/doe", "ref": "sha1-0123456789012345678901234567890123456789"},
map[string]string{"spec": "mem::sha1-0123456789012345678901234567890123456789", "scheme": "mem", "ref": "sha1-0123456789012345678901234567890123456789"},
}
for _, tc := range testCases {
refSpec, err := ParseRefSpec(tc["spec"])
assert.NoError(err)
dbSpec1 := DatabaseSpec{Protocol: tc["scheme"], Path: tc["path"], accessToken: tc["accessToken"]}
assert.Equal(RefSpec{DbSpec: dbSpec1, Ref: hash.Parse(tc["ref"])}, refSpec)
}
}
func TestPathSpec(t *testing.T) {
assert := assert.New(t)
testCases := []map[string]string{
map[string]string{"spec": "http://local.attic.io/john/doe::sha1-0123456789012345678901234567890123456789", "scheme": "http", "path": "//local.attic.io/john/doe", "ref": "sha1-0123456789012345678901234567890123456789"},
map[string]string{"spec": "http://localhost:8000/john/doe/::ds1", "scheme": "http", "path": "//localhost:8000/john/doe/", "ds": "ds1"},
}
for _, tc := range testCases {
pathSpec, err := ParsePathSpec(tc["spec"])
assert.NoError(err)
dbSpec1 := DatabaseSpec{Protocol: tc["scheme"], Path: tc["path"], accessToken: tc["accessToken"]}
if tc["ref"] != "" {
assert.Equal(&RefSpec{DbSpec: dbSpec1, Ref: hash.Parse(tc["ref"])}, pathSpec.(*RefSpec))
} else {
assert.Equal(&DatasetSpec{DbSpec: dbSpec1, DatasetName: tc["ds"]}, pathSpec.(*DatasetSpec))
}
}
_, err := ParsePathSpec("http://local.attic.io")
assert.Error(err)
}
func (r *nomsTestReader) readHash() hash.Hash {
return hash.Parse(r.readString())
}
func TestGetRequestBatch(t *testing.T) {
assert := assert.New(t)
r0 := hash.Parse("00000000000000000000000000000000")
c1 := NewChunk([]byte("abc"))
r1 := c1.Hash()
c2 := NewChunk([]byte("123"))
r2 := c2.Hash()
tally := func(b bool, trueCnt, falseCnt *int) {
if b {
*trueCnt++
} else {
*falseCnt++
}
}
req0chan := make(chan bool, 1)
req1chan := make(chan Chunk, 1)
req2chan := make(chan bool, 1)
req3chan := make(chan bool, 1)
req4chan := make(chan Chunk, 1)
batch := ReadBatch{
r0: []OutstandingRequest{OutstandingHas(req0chan), OutstandingGet(req1chan)},
r1: []OutstandingRequest{OutstandingHas(req2chan)},
r2: []OutstandingRequest{OutstandingHas(req3chan), OutstandingGet(req4chan)},
}
go func() {
for requestedRef, reqs := range batch {
for _, req := range reqs {
if requestedRef == r1 {
req.Satisfy(c1)
delete(batch, r1)
} else if requestedRef == r2 {
req.Satisfy(c2)
delete(batch, r2)
}
}
}
}()
var r1True, r1False, r2True, r2False int
for b := range req2chan {
tally(b, &r1True, &r1False)
}
for b := range req3chan {
tally(b, &r2True, &r2False)
}
for c := range req4chan {
assert.EqualValues(c2.Hash(), c.Hash())
}
assert.Equal(1, r1True)
assert.Equal(0, r1False)
assert.Equal(1, r2True)
assert.Equal(0, r2False)
go batch.Close()
var r0True, r0False int
for b := range req0chan {
tally(b, &r0True, &r0False)
}
for c := range req1chan {
assert.EqualValues(EmptyChunk.Hash(), c.Hash())
}
assert.Equal(0, r0True)
assert.Equal(1, r0False)
}
func (suite *ChunkStoreTestSuite) TestChunkStoreGetNonExisting() {
h := hash.Parse("11111111111111111111111111111111")
c := suite.Store.Get(h)
suite.True(c.IsEmpty())
}