Example #1
0
func TestMultiDuplicateContent(t *testing.T) {
	_, multi, _, done := prepareMultiTest(t, 1, 1, 1)
	defer done()

	data := []byte("this is some test data")

	for i := 0; i < 10; i++ {
		storetests.ShouldCAS(t, multi,
			strconv.FormatInt(int64(i), 10),
			store.MissingV, store.DataV(data))
	}

	for i := 0; i < 10; i++ {
		storetests.ShouldGet(t, multi,
			strconv.FormatInt(int64(i), 10),
			data)
	}

	for i := 0; i < 5; i++ {
		storetests.ShouldCAS(t, multi,
			strconv.FormatInt(int64(i), 10),
			store.AnyV, store.MissingV)
	}

	for i := 5; i < 10; i++ {
		storetests.ShouldGet(t, multi,
			strconv.FormatInt(int64(i), 10),
			data)
	}
}
Example #2
0
func TestMultiScrubRecreatesMissing(t *testing.T) {
	_, multi, mocks, done := prepareMultiTest(t, 2, 3, 3)
	defer done()

	data := []byte("hello world! this is some test data.")

	storetests.ShouldCAS(t, multi, "key", store.MissingV, store.DataV(data))

	names, err := mocks[0].List("", 1, nil)
	if err != nil {
		t.Fatalf("Couldn't list first mock: %v", err)
	}
	if len(names) != 1 {
		t.Fatalf("Didn't get a name from mock")
	}

	storetests.ShouldCAS(t, mocks[0], names[0], store.AnyV, store.MissingV)

	multi.scrubAll()

	found := 0
	for _, mock := range mocks {
		names, err := mock.List("", 1, nil)
		if err != nil {
			t.Fatalf("Couldn't list mock: %v", err)
		}
		if len(names) == 1 {
			found++
		}
	}

	if found != 3 {
		t.Fatalf("scrubAll didn't recreate missing chunk")
	}
}
Example #3
0
func TestMultiScrubRemovesUnreferencedChunks(t *testing.T) {
	killers, multi, mocks, done := prepareMultiTest(t, 1, 2, 2)
	defer done()

	storetests.ShouldCAS(t, multi, "a", store.MissingV, store.DataV([]byte("data")))
	killers[0].setKilled(true)
	storetests.ShouldCAS(t, multi, "a", store.DataV([]byte("data")), store.MissingV)
	killers[0].setKilled(false)
	storetests.ShouldListCount(t, multi, 0)
	storetests.ShouldListCount(t, mocks[0], 1)
	multi.finder.Rescan()
	multi.scrubAll()
	storetests.ShouldListCount(t, mocks[0], 0)
}
Example #4
0
func TestResplit(t *testing.T) {
	ds, tmpDir := makeTestingDirectory(t)
	defer os.RemoveAll(tmpDir)
	defer ds.Close()

	ds.mu.Lock()
	ds.minSplitSize = 2
	ds.maxSplitSize = 8
	ds.mu.Unlock()

	const itemCount = 50

	for i := 0; i < itemCount; i++ {
		key := strconv.FormatInt(int64(i), 3)
		storetests.ShouldCAS(t, ds, key, store.AnyV, store.DataV([]byte(key)))

		ds.resplit()

		for j := 0; j < itemCount; j++ {
			key := strconv.FormatInt(int64(j), 3)
			if j <= i {
				storetests.ShouldGet(t, ds, key, []byte(key))
			} else {
				storetests.ShouldGetMiss(t, ds, key)
			}
		}

		checkSplitCountSizes(t, "insertion", tmpDir, 2, 8)
	}

	for i := 0; i < itemCount; i++ {
		key := strconv.FormatInt(int64(i), 3)
		storetests.ShouldCAS(t, ds, key, store.AnyV, store.MissingV)

		ds.resplit()

		for j := 0; j < itemCount; j++ {
			key := strconv.FormatInt(int64(j), 3)
			if j <= i {
				storetests.ShouldGetMiss(t, ds, key)
			} else {
				storetests.ShouldGet(t, ds, key, []byte(key))
			}
		}

		checkSplitCountSizes(t, "deletion", tmpDir, 2, 8)
	}
}
Example #5
0
func TestMultiCanReplaceDeadKeys(t *testing.T) {
	killers, multi, _, done := prepareMultiTest(t, 3, 4, 4)
	defer done()

	storetests.ShouldCAS(t, multi, "a", store.MissingV, store.DataV([]byte("hello")))
	killers[0].setKilled(true)

	err := multi.SetRedundancy(1, 2)
	if err != nil {
		t.Fatalf("Couldn't adjust redundancy: %v", err)
	}

	storetests.ShouldCAS(t, multi, "a", store.DataV([]byte("hello")), store.DataV([]byte("there")))
	killers[1].setKilled(true)
	storetests.ShouldCAS(t, multi, "a", store.DataV([]byte("there")), store.MissingV)
}
Example #6
0
func TestMultiScrubChangeRedundancy(t *testing.T) {
	killers, multi, _, done := prepareMultiTest(t, 2, 3, 5)
	defer done()

	data := []byte("who knows where the wind goes")

	for i := 0; i < 10; i++ {
		storetests.ShouldCAS(t, multi,
			strconv.FormatInt(int64(i), 10),
			store.MissingV, store.DataV(data))
	}

	err := multi.SetRedundancy(2, 5)
	if err != nil {
		t.Fatalf("Couldn't adjust redundancy: %v", err)
	}

	multi.scrubAll()

	killers[0].setKilled(true)
	killers[1].setKilled(true)
	killers[2].setKilled(true)

	for i := 0; i < 10; i++ {
		storetests.ShouldGet(t, multi,
			strconv.FormatInt(int64(i), 10),
			data)
	}
}
Example #7
0
func TestMultiHungStoreDoesntBlock(t *testing.T) {
	killers, multi, _, done := prepareMultiTest(t, 2, 4, 4)
	defer done()

	oldTimeout := dataOnlyTimeout
	dataOnlyTimeout = 10 * time.Millisecond
	defer func() { dataOnlyTimeout = oldTimeout }()

	storetests.ShouldCAS(t, multi, "a", store.MissingV, store.DataV([]byte("data")))

	for _, k := range killers {
		k.setBlocked(true)

		result := make(chan error)
		go func() {
			_, _, err := multi.Get("a", store.GetOptions{})
			result <- err
		}()

		select {
		case err := <-result:
			if err != nil {
				t.Errorf("couldn't get key: %v", err)
			}
		case <-time.After(100 * time.Millisecond):
			t.Errorf("timed out waiting for read")
		}

		k.setBlocked(false)
	}
}
Example #8
0
func TestCacheGCWorks(t *testing.T) {
	inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
	cache := New(1024, inner)

	for i := 0; i < 1024; i++ {
		key := strconv.FormatInt(int64(i), 10)
		storetests.ShouldCAS(t, cache, key, store.AnyV, store.DataV([]byte(key)))
	}

	cache.assertUsedIsCorrect()
	t.Logf("cache.used after writes is %v", cache.used)

	for i := 0; i < 1024; i++ {
		key := strconv.FormatInt(int64(i), 10)
		storetests.ShouldGet(t, cache, key, []byte(key))
	}

	cache.assertUsedIsCorrect()
	t.Logf("cache.used after first reads is %v", cache.used)
	getsBefore := inner.gets

	for i := 0; i < 1024; i++ {
		key := strconv.FormatInt(int64(i), 10)
		storetests.ShouldGet(t, cache, key, []byte(key))
	}

	cache.assertUsedIsCorrect()
	t.Logf("cache.used after second reads is %v", cache.used)
	getsAfter := inner.gets

	if getsAfter != getsBefore+1024 {
		t.Errorf("wanted %v gets after, got %v", getsBefore+1024, getsAfter)
	}
}
Example #9
0
func TestMultiScrubRemovesWeirdChunks(t *testing.T) {
	_, multi, mocks, done := prepareMultiTest(t, 1, 1, 1)
	defer done()

	storetests.ShouldCAS(t, mocks[0], "a", store.MissingV, store.DataV([]byte("data")))
	multi.scrubAll()
	storetests.ShouldGetMiss(t, mocks[0], "a")
}
Example #10
0
func TestDirectoryCorruption(t *testing.T) {
	ds, tmpDir := makeTestingDirectory(t)
	defer os.RemoveAll(tmpDir)
	defer ds.Close()

	storetests.ShouldCAS(t, ds, "hello", store.AnyV, store.DataV([]byte("world")))
	shouldHashcheck(t, ds, 1, 0)
	shouldCorrupt(t, filepath.Join(tmpDir, "data", "1", "aGVsbG8="))
	storetests.ShouldGetError(t, ds, "hello", ErrCorruptObject)
	storetests.ShouldGetMiss(t, ds, "hello")
	shouldFileExist(t, filepath.Join(tmpDir, "quarantine", "aGVsbG8="))

	storetests.ShouldCAS(t, ds, "other", store.AnyV, store.DataV([]byte("werld")))
	shouldHashcheck(t, ds, 1, 0)
	shouldCorrupt(t, filepath.Join(tmpDir, "data", "1", "b3RoZXI="))
	shouldHashcheck(t, ds, 0, 1)
	storetests.ShouldGetMiss(t, ds, "other")
	shouldHashcheck(t, ds, 0, 0)
	shouldFileExist(t, filepath.Join(tmpDir, "quarantine", "b3RoZXI="))
}
Example #11
0
func TestCacheUpdatesOnUnknownChanges(t *testing.T) {
	inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
	cache := New(1024, inner)

	storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
	cache.Clear()
	storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
	storetests.ShouldCAS(t, inner, "asdf", store.AnyV, store.DataV([]byte("world")))
	storetests.ShouldGet(t, cache, "asdf", []byte("world"))

	if inner.gets != 2 {
		t.Errorf("wanted 2 inner gets, got %v", inner.gets)
	}

	if inner.stats != 1 {
		t.Errorf("wanted 1 inner Stats, got %v", inner.stats)
	}

	cache.assertUsedIsCorrect()
}
Example #12
0
func TestCacheUpdatesOnStats(t *testing.T) {
	inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
	cache := New(1024, inner)

	storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
	storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
	newData := store.DataV([]byte("world"))
	storetests.ShouldCAS(t, inner, "asdf", store.AnyV, newData)
	storetests.ShouldStatNoTime(t, cache, "asdf", store.Stat{Size: 5, SHA256: newData.SHA256})
	storetests.ShouldGet(t, cache, "asdf", []byte("world"))

	if inner.gets != 1 {
		t.Errorf("wanted 1 inner gets, got %v", inner.gets)
	}

	if inner.stats != 2 {
		t.Errorf("wanted 2 inner Stats, got %v", inner.stats)
	}

	cache.assertUsedIsCorrect()
}
Example #13
0
func TestHTTPRange(t *testing.T) {
	mock := storetests.NewMockStore(0)

	srv := NewServer(mock)

	storetests.ShouldCAS(t, mock, "key", store.AnyV, store.DataV([]byte("hello world!")))

	tests := []struct {
		RangeRequest   string
		Body           string
		RangeResponse  string
		NotSatisfiable bool
	}{
		{"bytes=5-", " world!", "bytes 5-11/12", false},
		{"bytes=5-12", " world!", "bytes 5-11/12", false},
		{"bytes=11-", "!", "bytes 11-11/12", false},
		{"bytes=0-", "hello world!", "bytes 0-11/12", false},
		{"bytes=1-1", "e", "bytes 1-1/12", false},
		{"bytes=4-4", "o", "bytes 4-4/12", false},
		{"bytes=12-", "", "", true},
		{"bytes=12341234-44", "", "", true},
	}

	for _, test := range tests {
		w := httptest.NewRecorder()
		r, err := http.NewRequest("GET", "/key", nil)
		if err != nil {
			t.Fatal(err)
		}
		r.Header.Set("Range", test.RangeRequest)
		srv.ServeHTTP(w, r)

		if test.NotSatisfiable {
			if w.Code != http.StatusRequestedRangeNotSatisfiable {
				t.Errorf("Wanted code %#v in response to %#v, got %#v",
					http.StatusRequestedRangeNotSatisfiable, test.RangeRequest,
					w.Code)
			}
		} else {
			if w.Body.String() != test.Body {
				t.Errorf("Wanted %#v in response to %#v, got %#v",
					test.Body, test.RangeRequest, w.Body.String())
			}
			if w.HeaderMap.Get("Content-Range") != test.RangeResponse {
				t.Errorf(`Wanted Content-Range %#v in response to %#v, got %#v`,
					test.RangeResponse, test.RangeRequest,
					w.HeaderMap.Get("Content-Range"))
			}
		}
	}
}
Example #14
0
func TestCacheDoesntCacheErrors(t *testing.T) {
	inner := &ErrorStore{MockStore: storetests.NewMockStore(0)}
	cache := New(1024, inner)

	storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
	cache.Clear()

	inner.isErroring = true
	for i := 0; i < 10; i++ {
		storetests.ShouldGetError(t, cache, "asdf", ErrErrorStore)
	}

	cache.assertUsedIsCorrect()

	inner.isErroring = false
	storetests.ShouldGet(t, cache, "asdf", []byte("hello"))

	cache.assertUsedIsCorrect()
}
Example #15
0
func TestCacheCachesGets(t *testing.T) {
	inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
	cache := New(1024, inner)

	storetests.ShouldCAS(t, inner, "asdf", store.AnyV, store.DataV([]byte("hello")))
	cache.assertUsedIsCorrect()

	for i := 0; i < 10; i++ {
		storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
		cache.assertUsedIsCorrect()
	}

	if inner.gets != 1 {
		t.Errorf("wanted 1 inner Get, got %v", inner.gets)
	}

	if inner.stats != 9 {
		t.Errorf("wanted 9 inner Stats, got %v", inner.stats)
	}

	cache.assertUsedIsCorrect()
}
Example #16
0
func TestMultiRecovery(t *testing.T) {
	for total := 4; total < 6; total++ {
		killers, multi, _, done := prepareMultiTest(t, 3, total, 6)
		defer done()

		for i := 0; i < 20; i++ {
			key := strconv.FormatInt(int64(i), 10)
			var value []byte
			for j := 0; j < i; j++ {
				value = append(value, []byte(key)...)
			}

			storetests.ShouldCAS(t, multi, key, store.MissingV, store.DataV(value))
		}

		for i := 0; i < total-3; i++ {
			for {
				k := killers[rand.Intn(len(killers))]
				if k.isKilled() {
					continue
				}
				k.setKilled(true)
				break
			}
		}

		for i := 0; i < 20; i++ {
			key := strconv.FormatInt(int64(i), 10)
			var value []byte
			for j := 0; j < i; j++ {
				value = append(value, []byte(key)...)
			}

			storetests.ShouldGet(t, multi, key, value)
		}
	}
}
Example #17
0
func TestHTTPConditionalGetLastModified(t *testing.T) {
	mock := storetests.NewMockStore(0)
	srv := NewServer(mock)

	storetests.ShouldCAS(t, mock, "key", store.AnyV, store.DataV([]byte("some data")))

	mkReq := func(ims string) *http.Request {
		req, err := http.NewRequest("GET", "/key", nil)
		if err != nil {
			t.Fatalf("Couldn't create request: %v", err)
		}
		if ims != "" {
			req.Header.Set("If-Modified-Since", ims)
		}
		return req
	}

	// First request: no special headers
	resp := httptest.NewRecorder()
	req := mkReq("")
	srv.ServeHTTP(resp, req)
	if resp.Code != 200 {
		t.Fatalf("Couldn't GET /key, status code %v", resp.Code)
	}

	lmStr := resp.HeaderMap.Get("Last-Modified")
	if lmStr == "" {
		t.Fatalf("Last-Modified header was not returned from GET")
	}

	lm, err := time.Parse(http.TimeFormat, lmStr)
	if err != nil {
		t.Fatalf("Couldn't parse Last-Modified %#v: %v", lmStr, err)
	}

	// Second request: matching If-Modified-Since time
	resp = httptest.NewRecorder()
	req = mkReq(lm.Format(http.TimeFormat))
	srv.ServeHTTP(resp, req)
	if resp.Code != 304 {
		t.Fatalf("Equal If-Modified-Since response did not return partial, got status %v",
			resp.Code)
	}

	// Third request: If-Modified-Since time in the past
	resp = httptest.NewRecorder()
	req = mkReq(lm.Add(-time.Second).Format(http.TimeFormat))
	srv.ServeHTTP(resp, req)
	if resp.Code != 200 {
		t.Fatalf("Past If-Modified-Since response did not return full, got status %v",
			resp.Code)
	}

	// Fourth request: If-Modified-Since time in the future
	resp = httptest.NewRecorder()
	req = mkReq(lm.Add(time.Second).Format(http.TimeFormat))
	srv.ServeHTTP(resp, req)
	if resp.Code != 304 {
		t.Fatalf("Equal If-Modified-Since response did not return partial, got status %v",
			resp.Code)
	}
}