func TestBuildWriteValueRequest(t *testing.T) {
assert := assert.New(t)
input1, input2 := "abc", "def"
chnx := []chunks.Chunk{
chunks.NewChunk([]byte(input1)),
chunks.NewChunk([]byte(input2)),
}
hints := map[hash.Hash]struct{}{
hash.Parse("sha1-0000000000000000000000000000000000000002"): struct{}{},
hash.Parse("sha1-0000000000000000000000000000000000000003"): struct{}{},
}
compressed := buildWriteValueRequest(serializeChunks(chnx, assert), hints)
gr := snappy.NewReader(compressed)
count := 0
for hint := range deserializeHints(gr) {
count++
_, present := hints[hint]
assert.True(present)
}
assert.Equal(len(hints), count)
chunkChan := make(chan *chunks.Chunk, 16)
go chunks.DeserializeToChan(gr, chunkChan)
for c := range chunkChan {
assert.Equal(chnx[0].Hash(), c.Hash())
chnx = chnx[1:]
}
assert.Empty(chnx)
}
func TestHandleGetRefs(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewTestStore()
input1, input2 := "abc", "def"
chnx := []chunks.Chunk{
chunks.NewChunk([]byte(input1)),
chunks.NewChunk([]byte(input2)),
}
err := cs.PutMany(chnx)
assert.NoError(err)
body := strings.NewReader(fmt.Sprintf("ref=%s&ref=%s", chnx[0].Hash(), chnx[1].Hash()))
w := httptest.NewRecorder()
HandleGetRefs(w,
&http.Request{Body: ioutil.NopCloser(body), Method: "POST", Header: http.Header{
"Content-Type": {"application/x-www-form-urlencoded"},
}},
params{},
cs,
)
if assert.Equal(http.StatusOK, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) {
chunkChan := make(chan *chunks.Chunk)
go chunks.DeserializeToChan(w.Body, chunkChan)
for c := range chunkChan {
assert.Equal(chnx[0].Hash(), c.Hash())
chnx = chnx[1:]
}
assert.Empty(chnx)
}
}
func (suite *LevelDBPutCacheSuite) extractChunks(hashes hashSet) <-chan *chunks.Chunk {
buf := &bytes.Buffer{}
err := suite.cache.ExtractChunks(hashes, buf)
suite.NoError(err)
chunkChan := make(chan *chunks.Chunk)
go chunks.DeserializeToChan(snappy.NewReader(buf), chunkChan)
return chunkChan
}
// Get can be called from any goroutine to retrieve the chunk referenced by hash. If the chunk is not present, Get returns the empty Chunk.
func (p *orderedChunkCache) Get(hash hash.Hash) chunks.Chunk {
// Don't use defer p.mu.RUnlock() here, because I want reading from orderedChunks NOT to be guarded by the lock. LevelDB handles its own goroutine-safety.
p.mu.RLock()
dbKey, ok := p.chunkIndex[hash]
p.mu.RUnlock()
if !ok {
return chunks.EmptyChunk
}
data, err := p.orderedChunks.Get(dbKey, nil)
d.Chk.NoError(err)
reader := snappy.NewReader(bytes.NewReader(data))
chunkChan := make(chan *chunks.Chunk)
go chunks.DeserializeToChan(reader, chunkChan)
return *(<-chunkChan)
}
func HandleWriteValue(w http.ResponseWriter, req *http.Request, ps URLParams, cs chunks.ChunkStore) {
hashes := hash.HashSlice{}
err := d.Try(func() {
d.Exp.Equal("POST", req.Method)
reader := bodyReader(req)
defer func() {
// Ensure all data on reader is consumed
io.Copy(ioutil.Discard, reader)
reader.Close()
}()
vbs := types.NewValidatingBatchingSink(cs)
vbs.Prepare(deserializeHints(reader))
chunkChan := make(chan *chunks.Chunk, 16)
go chunks.DeserializeToChan(reader, chunkChan)
var bpe chunks.BackpressureError
for c := range chunkChan {
if bpe == nil {
bpe = vbs.Enqueue(*c)
} else {
bpe = append(bpe, c.Hash())
}
// If a previous Enqueue() errored, we still need to drain chunkChan
// TODO: what about having DeserializeToChan take a 'done' channel to stop it?
hashes = append(hashes, c.Hash())
}
if bpe == nil {
bpe = vbs.Flush()
}
if bpe != nil {
w.WriteHeader(httpStatusTooManyRequests)
w.Header().Add("Content-Type", "application/octet-stream")
writer := respWriter(req, w)
defer writer.Close()
serializeHashes(writer, bpe.AsHashes())
return
}
w.WriteHeader(http.StatusCreated)
})
if err != nil {
http.Error(w, fmt.Sprintf("Error: %v\nChunks in payload: %v", err, hashes), http.StatusBadRequest)
return
}
}
func handleWriteValue(w http.ResponseWriter, req *http.Request, ps URLParams, cs chunks.ChunkStore) {
d.PanicIfTrue(req.Method != "POST", "Expected post method.")
reader := bodyReader(req)
defer func() {
// Ensure all data on reader is consumed
io.Copy(ioutil.Discard, reader)
reader.Close()
}()
tc := types.NewTypeCache()
vbs := types.NewValidatingBatchingSink(cs, tc)
vbs.Prepare(deserializeHints(reader))
chunkChan := make(chan *chunks.Chunk, 16)
go chunks.DeserializeToChan(reader, chunkChan)
var bpe chunks.BackpressureError
for c := range chunkChan {
if bpe == nil {
bpe = vbs.Enqueue(*c)
} else {
bpe = append(bpe, c.Hash())
}
// If a previous Enqueue() errored, we still need to drain chunkChan
// TODO: what about having DeserializeToChan take a 'done' channel to stop it?
}
if bpe == nil {
bpe = vbs.Flush()
}
if bpe != nil {
w.WriteHeader(httpStatusTooManyRequests)
w.Header().Add("Content-Type", "application/octet-stream")
writer := respWriter(req, w)
defer writer.Close()
serializeHashes(writer, bpe.AsHashes())
return
}
w.WriteHeader(http.StatusCreated)
}
