func TestMultiDuplicateContent(t *testing.T) {
_, multi, _, done := prepareMultiTest(t, 1, 1, 1)
defer done()
data := []byte("this is some test data")
for i := 0; i < 10; i++ {
storetests.ShouldCAS(t, multi,
strconv.FormatInt(int64(i), 10),
store.MissingV, store.DataV(data))
}
for i := 0; i < 10; i++ {
storetests.ShouldGet(t, multi,
strconv.FormatInt(int64(i), 10),
data)
}
for i := 0; i < 5; i++ {
storetests.ShouldCAS(t, multi,
strconv.FormatInt(int64(i), 10),
store.AnyV, store.MissingV)
}
for i := 5; i < 10; i++ {
storetests.ShouldGet(t, multi,
strconv.FormatInt(int64(i), 10),
data)
}
}
func TestMultiScrubRecreatesMissing(t *testing.T) {
_, multi, mocks, done := prepareMultiTest(t, 2, 3, 3)
defer done()
data := []byte("hello world! this is some test data.")
storetests.ShouldCAS(t, multi, "key", store.MissingV, store.DataV(data))
names, err := mocks[0].List("", 1, nil)
if err != nil {
t.Fatalf("Couldn't list first mock: %v", err)
}
if len(names) != 1 {
t.Fatalf("Didn't get a name from mock")
}
storetests.ShouldCAS(t, mocks[0], names[0], store.AnyV, store.MissingV)
multi.scrubAll()
found := 0
for _, mock := range mocks {
names, err := mock.List("", 1, nil)
if err != nil {
t.Fatalf("Couldn't list mock: %v", err)
}
if len(names) == 1 {
found++
}
}
if found != 3 {
t.Fatalf("scrubAll didn't recreate missing chunk")
}
}
func TestMultiScrubRemovesUnreferencedChunks(t *testing.T) {
killers, multi, mocks, done := prepareMultiTest(t, 1, 2, 2)
defer done()
storetests.ShouldCAS(t, multi, "a", store.MissingV, store.DataV([]byte("data")))
killers[0].setKilled(true)
storetests.ShouldCAS(t, multi, "a", store.DataV([]byte("data")), store.MissingV)
killers[0].setKilled(false)
storetests.ShouldListCount(t, multi, 0)
storetests.ShouldListCount(t, mocks[0], 1)
multi.finder.Rescan()
multi.scrubAll()
storetests.ShouldListCount(t, mocks[0], 0)
}
func TestResplit(t *testing.T) {
ds, tmpDir := makeTestingDirectory(t)
defer os.RemoveAll(tmpDir)
defer ds.Close()
ds.mu.Lock()
ds.minSplitSize = 2
ds.maxSplitSize = 8
ds.mu.Unlock()
const itemCount = 50
for i := 0; i < itemCount; i++ {
key := strconv.FormatInt(int64(i), 3)
storetests.ShouldCAS(t, ds, key, store.AnyV, store.DataV([]byte(key)))
ds.resplit()
for j := 0; j < itemCount; j++ {
key := strconv.FormatInt(int64(j), 3)
if j <= i {
storetests.ShouldGet(t, ds, key, []byte(key))
} else {
storetests.ShouldGetMiss(t, ds, key)
}
}
checkSplitCountSizes(t, "insertion", tmpDir, 2, 8)
}
for i := 0; i < itemCount; i++ {
key := strconv.FormatInt(int64(i), 3)
storetests.ShouldCAS(t, ds, key, store.AnyV, store.MissingV)
ds.resplit()
for j := 0; j < itemCount; j++ {
key := strconv.FormatInt(int64(j), 3)
if j <= i {
storetests.ShouldGetMiss(t, ds, key)
} else {
storetests.ShouldGet(t, ds, key, []byte(key))
}
}
checkSplitCountSizes(t, "deletion", tmpDir, 2, 8)
}
}
func TestMultiCanReplaceDeadKeys(t *testing.T) {
killers, multi, _, done := prepareMultiTest(t, 3, 4, 4)
defer done()
storetests.ShouldCAS(t, multi, "a", store.MissingV, store.DataV([]byte("hello")))
killers[0].setKilled(true)
err := multi.SetRedundancy(1, 2)
if err != nil {
t.Fatalf("Couldn't adjust redundancy: %v", err)
}
storetests.ShouldCAS(t, multi, "a", store.DataV([]byte("hello")), store.DataV([]byte("there")))
killers[1].setKilled(true)
storetests.ShouldCAS(t, multi, "a", store.DataV([]byte("there")), store.MissingV)
}
func TestMultiScrubChangeRedundancy(t *testing.T) {
killers, multi, _, done := prepareMultiTest(t, 2, 3, 5)
defer done()
data := []byte("who knows where the wind goes")
for i := 0; i < 10; i++ {
storetests.ShouldCAS(t, multi,
strconv.FormatInt(int64(i), 10),
store.MissingV, store.DataV(data))
}
err := multi.SetRedundancy(2, 5)
if err != nil {
t.Fatalf("Couldn't adjust redundancy: %v", err)
}
multi.scrubAll()
killers[0].setKilled(true)
killers[1].setKilled(true)
killers[2].setKilled(true)
for i := 0; i < 10; i++ {
storetests.ShouldGet(t, multi,
strconv.FormatInt(int64(i), 10),
data)
}
}
func TestMultiHungStoreDoesntBlock(t *testing.T) {
killers, multi, _, done := prepareMultiTest(t, 2, 4, 4)
defer done()
oldTimeout := dataOnlyTimeout
dataOnlyTimeout = 10 * time.Millisecond
defer func() { dataOnlyTimeout = oldTimeout }()
storetests.ShouldCAS(t, multi, "a", store.MissingV, store.DataV([]byte("data")))
for _, k := range killers {
k.setBlocked(true)
result := make(chan error)
go func() {
_, _, err := multi.Get("a", store.GetOptions{})
result <- err
}()
select {
case err := <-result:
if err != nil {
t.Errorf("couldn't get key: %v", err)
}
case <-time.After(100 * time.Millisecond):
t.Errorf("timed out waiting for read")
}
k.setBlocked(false)
}
}
func TestCacheGCWorks(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
for i := 0; i < 1024; i++ {
key := strconv.FormatInt(int64(i), 10)
storetests.ShouldCAS(t, cache, key, store.AnyV, store.DataV([]byte(key)))
}
cache.assertUsedIsCorrect()
t.Logf("cache.used after writes is %v", cache.used)
for i := 0; i < 1024; i++ {
key := strconv.FormatInt(int64(i), 10)
storetests.ShouldGet(t, cache, key, []byte(key))
}
cache.assertUsedIsCorrect()
t.Logf("cache.used after first reads is %v", cache.used)
getsBefore := inner.gets
for i := 0; i < 1024; i++ {
key := strconv.FormatInt(int64(i), 10)
storetests.ShouldGet(t, cache, key, []byte(key))
}
cache.assertUsedIsCorrect()
t.Logf("cache.used after second reads is %v", cache.used)
getsAfter := inner.gets
if getsAfter != getsBefore+1024 {
t.Errorf("wanted %v gets after, got %v", getsBefore+1024, getsAfter)
}
}
func TestMultiScrubRemovesWeirdChunks(t *testing.T) {
_, multi, mocks, done := prepareMultiTest(t, 1, 1, 1)
defer done()
storetests.ShouldCAS(t, mocks[0], "a", store.MissingV, store.DataV([]byte("data")))
multi.scrubAll()
storetests.ShouldGetMiss(t, mocks[0], "a")
}
func TestDirectoryCorruption(t *testing.T) {
ds, tmpDir := makeTestingDirectory(t)
defer os.RemoveAll(tmpDir)
defer ds.Close()
storetests.ShouldCAS(t, ds, "hello", store.AnyV, store.DataV([]byte("world")))
shouldHashcheck(t, ds, 1, 0)
shouldCorrupt(t, filepath.Join(tmpDir, "data", "1", "aGVsbG8="))
storetests.ShouldGetError(t, ds, "hello", ErrCorruptObject)
storetests.ShouldGetMiss(t, ds, "hello")
shouldFileExist(t, filepath.Join(tmpDir, "quarantine", "aGVsbG8="))
storetests.ShouldCAS(t, ds, "other", store.AnyV, store.DataV([]byte("werld")))
shouldHashcheck(t, ds, 1, 0)
shouldCorrupt(t, filepath.Join(tmpDir, "data", "1", "b3RoZXI="))
shouldHashcheck(t, ds, 0, 1)
storetests.ShouldGetMiss(t, ds, "other")
shouldHashcheck(t, ds, 0, 0)
shouldFileExist(t, filepath.Join(tmpDir, "quarantine", "b3RoZXI="))
}
func TestCacheUpdatesOnUnknownChanges(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
cache.Clear()
storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
storetests.ShouldCAS(t, inner, "asdf", store.AnyV, store.DataV([]byte("world")))
storetests.ShouldGet(t, cache, "asdf", []byte("world"))
if inner.gets != 2 {
t.Errorf("wanted 2 inner gets, got %v", inner.gets)
}
if inner.stats != 1 {
t.Errorf("wanted 1 inner Stats, got %v", inner.stats)
}
cache.assertUsedIsCorrect()
}
func TestCacheUpdatesOnStats(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
newData := store.DataV([]byte("world"))
storetests.ShouldCAS(t, inner, "asdf", store.AnyV, newData)
storetests.ShouldStatNoTime(t, cache, "asdf", store.Stat{Size: 5, SHA256: newData.SHA256})
storetests.ShouldGet(t, cache, "asdf", []byte("world"))
if inner.gets != 1 {
t.Errorf("wanted 1 inner gets, got %v", inner.gets)
}
if inner.stats != 2 {
t.Errorf("wanted 2 inner Stats, got %v", inner.stats)
}
cache.assertUsedIsCorrect()
}
func TestHTTPRange(t *testing.T) {
mock := storetests.NewMockStore(0)
srv := NewServer(mock)
storetests.ShouldCAS(t, mock, "key", store.AnyV, store.DataV([]byte("hello world!")))
tests := []struct {
RangeRequest string
Body string
RangeResponse string
NotSatisfiable bool
}{
{"bytes=5-", " world!", "bytes 5-11/12", false},
{"bytes=5-12", " world!", "bytes 5-11/12", false},
{"bytes=11-", "!", "bytes 11-11/12", false},
{"bytes=0-", "hello world!", "bytes 0-11/12", false},
{"bytes=1-1", "e", "bytes 1-1/12", false},
{"bytes=4-4", "o", "bytes 4-4/12", false},
{"bytes=12-", "", "", true},
{"bytes=12341234-44", "", "", true},
}
for _, test := range tests {
w := httptest.NewRecorder()
r, err := http.NewRequest("GET", "/key", nil)
if err != nil {
t.Fatal(err)
}
r.Header.Set("Range", test.RangeRequest)
srv.ServeHTTP(w, r)
if test.NotSatisfiable {
if w.Code != http.StatusRequestedRangeNotSatisfiable {
t.Errorf("Wanted code %#v in response to %#v, got %#v",
http.StatusRequestedRangeNotSatisfiable, test.RangeRequest,
w.Code)
}
} else {
if w.Body.String() != test.Body {
t.Errorf("Wanted %#v in response to %#v, got %#v",
test.Body, test.RangeRequest, w.Body.String())
}
if w.HeaderMap.Get("Content-Range") != test.RangeResponse {
t.Errorf(`Wanted Content-Range %#v in response to %#v, got %#v`,
test.RangeResponse, test.RangeRequest,
w.HeaderMap.Get("Content-Range"))
}
}
}
}
func TestCacheDoesntCacheErrors(t *testing.T) {
inner := &ErrorStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
cache.Clear()
inner.isErroring = true
for i := 0; i < 10; i++ {
storetests.ShouldGetError(t, cache, "asdf", ErrErrorStore)
}
cache.assertUsedIsCorrect()
inner.isErroring = false
storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
cache.assertUsedIsCorrect()
}
func TestCacheCachesGets(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
storetests.ShouldCAS(t, inner, "asdf", store.AnyV, store.DataV([]byte("hello")))
cache.assertUsedIsCorrect()
for i := 0; i < 10; i++ {
storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
cache.assertUsedIsCorrect()
}
if inner.gets != 1 {
t.Errorf("wanted 1 inner Get, got %v", inner.gets)
}
if inner.stats != 9 {
t.Errorf("wanted 9 inner Stats, got %v", inner.stats)
}
cache.assertUsedIsCorrect()
}
func TestMultiRecovery(t *testing.T) {
for total := 4; total < 6; total++ {
killers, multi, _, done := prepareMultiTest(t, 3, total, 6)
defer done()
for i := 0; i < 20; i++ {
key := strconv.FormatInt(int64(i), 10)
var value []byte
for j := 0; j < i; j++ {
value = append(value, []byte(key)...)
}
storetests.ShouldCAS(t, multi, key, store.MissingV, store.DataV(value))
}
for i := 0; i < total-3; i++ {
for {
k := killers[rand.Intn(len(killers))]
if k.isKilled() {
continue
}
k.setKilled(true)
break
}
}
for i := 0; i < 20; i++ {
key := strconv.FormatInt(int64(i), 10)
var value []byte
for j := 0; j < i; j++ {
value = append(value, []byte(key)...)
}
storetests.ShouldGet(t, multi, key, value)
}
}
}
func TestHTTPConditionalGetLastModified(t *testing.T) {
mock := storetests.NewMockStore(0)
srv := NewServer(mock)
storetests.ShouldCAS(t, mock, "key", store.AnyV, store.DataV([]byte("some data")))
mkReq := func(ims string) *http.Request {
req, err := http.NewRequest("GET", "/key", nil)
if err != nil {
t.Fatalf("Couldn't create request: %v", err)
}
if ims != "" {
req.Header.Set("If-Modified-Since", ims)
}
return req
}
// First request: no special headers
resp := httptest.NewRecorder()
req := mkReq("")
srv.ServeHTTP(resp, req)
if resp.Code != 200 {
t.Fatalf("Couldn't GET /key, status code %v", resp.Code)
}
lmStr := resp.HeaderMap.Get("Last-Modified")
if lmStr == "" {
t.Fatalf("Last-Modified header was not returned from GET")
}
lm, err := time.Parse(http.TimeFormat, lmStr)
if err != nil {
t.Fatalf("Couldn't parse Last-Modified %#v: %v", lmStr, err)
}
// Second request: matching If-Modified-Since time
resp = httptest.NewRecorder()
req = mkReq(lm.Format(http.TimeFormat))
srv.ServeHTTP(resp, req)
if resp.Code != 304 {
t.Fatalf("Equal If-Modified-Since response did not return partial, got status %v",
resp.Code)
}
// Third request: If-Modified-Since time in the past
resp = httptest.NewRecorder()
req = mkReq(lm.Add(-time.Second).Format(http.TimeFormat))
srv.ServeHTTP(resp, req)
if resp.Code != 200 {
t.Fatalf("Past If-Modified-Since response did not return full, got status %v",
resp.Code)
}
// Fourth request: If-Modified-Since time in the future
resp = httptest.NewRecorder()
req = mkReq(lm.Add(time.Second).Format(http.TimeFormat))
srv.ServeHTTP(resp, req)
if resp.Code != 304 {
t.Fatalf("Equal If-Modified-Since response did not return partial, got status %v",
resp.Code)
}
}
