func TestCacheGCWorks(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
for i := 0; i < 1024; i++ {
key := strconv.FormatInt(int64(i), 10)
storetests.ShouldCAS(t, cache, key, store.AnyV, store.DataV([]byte(key)))
}
cache.assertUsedIsCorrect()
t.Logf("cache.used after writes is %v", cache.used)
for i := 0; i < 1024; i++ {
key := strconv.FormatInt(int64(i), 10)
storetests.ShouldGet(t, cache, key, []byte(key))
}
cache.assertUsedIsCorrect()
t.Logf("cache.used after first reads is %v", cache.used)
getsBefore := inner.gets
for i := 0; i < 1024; i++ {
key := strconv.FormatInt(int64(i), 10)
storetests.ShouldGet(t, cache, key, []byte(key))
}
cache.assertUsedIsCorrect()
t.Logf("cache.used after second reads is %v", cache.used)
getsAfter := inner.gets
if getsAfter != getsBefore+1024 {
t.Errorf("wanted %v gets after, got %v", getsBefore+1024, getsAfter)
}
}
func TestCacheCoalescesGets(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
inner.SetBlocked(true)
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
cache.Get("key", store.GetOptions{})
wg.Done()
}()
}
// wait until Gets are in flight (unreliable)
time.Sleep(20 * time.Millisecond)
inner.SetBlocked(false)
wg.Wait()
if inner.gets != 1 {
t.Errorf("wanted 1 inner Get, got %v", inner.gets)
}
if inner.stats != 0 {
t.Errorf("wanted 0 inner Stats, got %v", inner.stats)
}
}
func TestCacheGeneric(t *testing.T) {
for size := 1; size <= 1024*128; size *= 4 {
inner := storetests.NewMockStore(0)
cache := New(size, inner)
storetests.TestStore(t, cache)
cache.assertUsedIsCorrect()
}
}
func TestHTTPRange(t *testing.T) {
mock := storetests.NewMockStore(0)
srv := NewServer(mock)
storetests.ShouldCAS(t, mock, "key", store.AnyV, store.DataV([]byte("hello world!")))
tests := []struct {
RangeRequest string
Body string
RangeResponse string
NotSatisfiable bool
}{
{"bytes=5-", " world!", "bytes 5-11/12", false},
{"bytes=5-12", " world!", "bytes 5-11/12", false},
{"bytes=11-", "!", "bytes 11-11/12", false},
{"bytes=0-", "hello world!", "bytes 0-11/12", false},
{"bytes=1-1", "e", "bytes 1-1/12", false},
{"bytes=4-4", "o", "bytes 4-4/12", false},
{"bytes=12-", "", "", true},
{"bytes=12341234-44", "", "", true},
}
for _, test := range tests {
w := httptest.NewRecorder()
r, err := http.NewRequest("GET", "/key", nil)
if err != nil {
t.Fatal(err)
}
r.Header.Set("Range", test.RangeRequest)
srv.ServeHTTP(w, r)
if test.NotSatisfiable {
if w.Code != http.StatusRequestedRangeNotSatisfiable {
t.Errorf("Wanted code %#v in response to %#v, got %#v",
http.StatusRequestedRangeNotSatisfiable, test.RangeRequest,
w.Code)
}
} else {
if w.Body.String() != test.Body {
t.Errorf("Wanted %#v in response to %#v, got %#v",
test.Body, test.RangeRequest, w.Body.String())
}
if w.HeaderMap.Get("Content-Range") != test.RangeResponse {
t.Errorf(`Wanted Content-Range %#v in response to %#v, got %#v`,
test.RangeResponse, test.RangeRequest,
w.HeaderMap.Get("Content-Range"))
}
}
}
}
func TestHTTPCommon(t *testing.T) {
mock := storetests.NewMockStore(0)
srv := httptest.NewServer(NewServer(mock))
defer srv.Close()
client, err := NewClient(srv.URL + "/")
if err != nil {
t.Fatalf("Couldn't initialize client: %v", err)
}
storetests.TestStore(t, client)
if client.UUID() != mock.UUID() {
t.Errorf("client UUID %v does not match directory UUID %v",
uuid.Fmt(client.UUID()), uuid.Fmt(mock.UUID()))
}
}
func TestCacheDoesntCacheErrors(t *testing.T) {
inner := &ErrorStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
cache.Clear()
inner.isErroring = true
for i := 0; i < 10; i++ {
storetests.ShouldGetError(t, cache, "asdf", ErrErrorStore)
}
cache.assertUsedIsCorrect()
inner.isErroring = false
storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
cache.assertUsedIsCorrect()
}
func TestCacheUpdatesOnUnknownChanges(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
cache.Clear()
storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
storetests.ShouldCAS(t, inner, "asdf", store.AnyV, store.DataV([]byte("world")))
storetests.ShouldGet(t, cache, "asdf", []byte("world"))
if inner.gets != 2 {
t.Errorf("wanted 2 inner gets, got %v", inner.gets)
}
if inner.stats != 1 {
t.Errorf("wanted 1 inner Stats, got %v", inner.stats)
}
cache.assertUsedIsCorrect()
}
func TestHTTPTooBig(t *testing.T) {
mock := storetests.NewMockStore(0)
handler := NewServer(mock)
data := make([]byte, MaxFileSize+1000)
w := httptest.NewRecorder()
req, err := http.NewRequest("PUT", "/thing", bytes.NewReader(data))
if err != nil {
t.Fatalf("Couldn't create request: %v", err)
}
handler.ServeHTTP(w, req)
if w.Code != http.StatusRequestEntityTooLarge {
t.Errorf("Wanted response code %v, but got %v",
http.StatusRequestEntityTooLarge, w.Code)
}
}
func TestCacheUpdatesOnStats(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
storetests.ShouldCAS(t, cache, "asdf", store.AnyV, store.DataV([]byte("hello")))
storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
newData := store.DataV([]byte("world"))
storetests.ShouldCAS(t, inner, "asdf", store.AnyV, newData)
storetests.ShouldStatNoTime(t, cache, "asdf", store.Stat{Size: 5, SHA256: newData.SHA256})
storetests.ShouldGet(t, cache, "asdf", []byte("world"))
if inner.gets != 1 {
t.Errorf("wanted 1 inner gets, got %v", inner.gets)
}
if inner.stats != 2 {
t.Errorf("wanted 2 inner Stats, got %v", inner.stats)
}
cache.assertUsedIsCorrect()
}
func TestHTTPNoVerify(t *testing.T) {
mock := storetests.NewMockStore(0)
recorder := &noverifyRecorder{mock, nil}
srv := httptest.NewServer(NewServer(recorder))
defer srv.Close()
client, err := NewClient(srv.URL + "/")
if err != nil {
t.Fatalf("Couldn't initialize client: %v", err)
}
pattern := []bool{false, true, false}
for _, nv := range pattern {
client.Get("key", store.GetOptions{NoVerify: nv})
}
if !reflect.DeepEqual(recorder.noverify, pattern) {
t.Fatalf("Wanted noverify pattern %#v, but got %#v", pattern, recorder.noverify)
}
}
func TestCacheCachesGets(t *testing.T) {
inner := &CountingStore{MockStore: storetests.NewMockStore(0)}
cache := New(1024, inner)
storetests.ShouldCAS(t, inner, "asdf", store.AnyV, store.DataV([]byte("hello")))
cache.assertUsedIsCorrect()
for i := 0; i < 10; i++ {
storetests.ShouldGet(t, cache, "asdf", []byte("hello"))
cache.assertUsedIsCorrect()
}
if inner.gets != 1 {
t.Errorf("wanted 1 inner Get, got %v", inner.gets)
}
if inner.stats != 9 {
t.Errorf("wanted 9 inner Stats, got %v", inner.stats)
}
cache.assertUsedIsCorrect()
}
func TestClientCancel(t *testing.T) {
mock := storetests.NewMockStore(0)
handler := NewServer(mock)
var serverSending chan struct{}
serverWait := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case serverSending <- struct{}{}:
default:
}
<-serverWait
handler.ServeHTTP(w, r)
}))
defer srv.Close()
startBlocking := make(chan struct{})
startedBlocking := make(chan struct{})
go func() {
defer close(startedBlocking)
for {
select {
case <-startBlocking:
return
case serverWait <- struct{}{}:
}
}
}()
client, err := NewClient(srv.URL + "/")
if err != nil {
t.Fatalf("Couldn't initialize client: %v", err)
}
defer client.Close()
close(startBlocking)
<-startedBlocking
serverSending = make(chan struct{})
cancel := make(chan struct{})
clientGet := make(chan error)
go func() {
_, _, err := client.Get("key", store.GetOptions{Cancel: cancel})
clientGet <- err
}()
<-serverSending // wait until the client has asked the server
close(cancel) // then cancel
select {
case err = <-clientGet:
case <-time.After(time.Second):
t.Fatalf("client cancel did not return early")
}
if err != store.ErrCancelled {
t.Errorf("cancelled client get did not return ErrCancelled (got %v)", err)
}
close(serverWait)
_, _, err = client.Get("key", store.GetOptions{})
if err != store.ErrNotFound {
t.Errorf("client did not recover from a cancelled request")
}
}
func TestHTTPConditionalGetLastModified(t *testing.T) {
mock := storetests.NewMockStore(0)
srv := NewServer(mock)
storetests.ShouldCAS(t, mock, "key", store.AnyV, store.DataV([]byte("some data")))
mkReq := func(ims string) *http.Request {
req, err := http.NewRequest("GET", "/key", nil)
if err != nil {
t.Fatalf("Couldn't create request: %v", err)
}
if ims != "" {
req.Header.Set("If-Modified-Since", ims)
}
return req
}
// First request: no special headers
resp := httptest.NewRecorder()
req := mkReq("")
srv.ServeHTTP(resp, req)
if resp.Code != 200 {
t.Fatalf("Couldn't GET /key, status code %v", resp.Code)
}
lmStr := resp.HeaderMap.Get("Last-Modified")
if lmStr == "" {
t.Fatalf("Last-Modified header was not returned from GET")
}
lm, err := time.Parse(http.TimeFormat, lmStr)
if err != nil {
t.Fatalf("Couldn't parse Last-Modified %#v: %v", lmStr, err)
}
// Second request: matching If-Modified-Since time
resp = httptest.NewRecorder()
req = mkReq(lm.Format(http.TimeFormat))
srv.ServeHTTP(resp, req)
if resp.Code != 304 {
t.Fatalf("Equal If-Modified-Since response did not return partial, got status %v",
resp.Code)
}
// Third request: If-Modified-Since time in the past
resp = httptest.NewRecorder()
req = mkReq(lm.Add(-time.Second).Format(http.TimeFormat))
srv.ServeHTTP(resp, req)
if resp.Code != 200 {
t.Fatalf("Past If-Modified-Since response did not return full, got status %v",
resp.Code)
}
// Fourth request: If-Modified-Since time in the future
resp = httptest.NewRecorder()
req = mkReq(lm.Add(time.Second).Format(http.TimeFormat))
srv.ServeHTTP(resp, req)
if resp.Code != 304 {
t.Fatalf("Equal If-Modified-Since response did not return partial, got status %v",
resp.Code)
}
}
func TestHTTPCancellation(t *testing.T) {
timeout := time.NewTimer(5 * time.Second)
defer timeout.Stop()
mock := storetests.NewMockStore(0)
recorder := &cancelRecorder{Store: mock}
srv := httptest.NewServer(NewServer(recorder))
defer srv.Close()
client, err := NewClient(srv.URL + "/")
if err != nil {
t.Fatalf("Couldn't initialize client: %v", err)
}
recorder.ClearCancels()
mock.SetBlocked(true)
defer mock.SetBlocked(false)
// Start a request
clientCancel := make(chan struct{})
clientErr := make(chan error, 1)
go func() {
_, _, err := client.Get("key", store.GetOptions{Cancel: clientCancel})
clientErr <- err
}()
// Wait until the request reaches the server
var serverCancel <-chan struct{}
for {
select {
case <-timeout.C:
t.Fatalf("Timed out while waiting for server to recieve request")
case <-time.After(5 * time.Millisecond):
}
cancels := recorder.Cancels()
if len(cancels) == 0 {
// Request hasn't reached the server yet
continue
}
if len(cancels) == 1 {
serverCancel = cancels[0]
break
}
t.Fatalf("Got multiple cancel channels")
}
// Close the client request
close(clientCancel)
// Wait for the client to respond to the cancellation
select {
case err := <-clientErr:
if err != store.ErrCancelled {
t.Errorf("Client Get returned err %v after canceling, wanted %v",
err, store.ErrCancelled)
}
case <-timeout.C:
t.Fatalf("Client Get did not return after cancellation")
}
// Wait for the server to respond to the cancellation
LOOP:
for {
select {
case <-serverCancel:
break LOOP
case <-timeout.C:
t.Fatalf("Timed out waiting for server cancel")
}
}
}
func TestCacheEmptyHasCorrectUsed(t *testing.T) {
cache := New(1024, storetests.NewMockStore(0))
cache.assertUsedIsCorrect()
}
func prepareMultiTest(t testing.TB, need, total, serverCount int) ([]*killHandler, *Multi, []*storetests.MockStore, func()) {
var killers []*killHandler
var mockstores []*storetests.MockStore
var servers []*httptest.Server
var chunkServers []*chunkserver.Handler
var finder *Finder
var multi *Multi
done := func() {
if multi != nil {
multi.Close()
}
if finder != nil {
finder.Stop()
}
for _, srv := range servers {
srv.Close()
}
for _, cs := range chunkServers {
cs.Close()
}
for _, mock := range mockstores {
mock.Close()
}
}
for i := 0; i < serverCount; i++ {
mock := storetests.NewMockStore(0)
mockstores = append(mockstores, mock)
cs, err := chunkserver.New([]store.Store{mock})
if err != nil {
done()
t.Fatalf("Couldn't create chunkserver: %v", err)
}
chunkServers = append(chunkServers, cs)
killer := newKillHandler(cs)
srv := httptest.NewServer(killer)
killers = append(killers, killer)
servers = append(servers, srv)
}
db := ram.New()
finder, err := NewFinder(db)
if err != nil {
done()
t.Fatalf("Couldn't create new finder: %v", err)
}
for _, srv := range servers {
err = finder.Scan(srv.URL)
if err != nil {
done()
t.Fatalf("Couldn't scan %v: %v", srv.URL, err)
}
}
if len(finder.Stores()) != len(servers) {
done()
t.Fatalf("Finder did not find all stores")
}
multi, err = NewMulti(db, finder, 0)
if err != nil {
done()
t.Fatalf("Couldn't create multi: %v", err)
}
err = multi.SetRedundancy(need, total)
if err != nil {
done()
t.Fatalf("Couldn't set redundancy levels: %v", err)
}
return killers, multi, mockstores, done
}
func TestFinderScan(t *testing.T) {
db := ram.New()
mock := storetests.NewMockStore(0)
cs, err := chunkserver.New([]store.Store{mock})
if err != nil {
t.Fatalf("Couldn't create chunkserver: %v", err)
}
defer cs.Close()
cs.WaitAllAvailable()
killer := newKillHandler(cs)
srv := httptest.NewServer(killer)
defer srv.Close()
f, err := NewFinder(db)
if err != nil {
t.Fatalf("Couldn't create new finder: %v", err)
}
defer f.Stop()
err = f.Scan(srv.URL)
if err != nil {
t.Fatalf("Couldn't scan %v: %v", srv.URL, err)
}
// newly scanned store should be in the Finder
stores := f.Stores()
if _, ok := stores[mock.UUID()]; !ok {
t.Fatalf("Finder did not find uuid of directory store")
}
// kill the store and update the Finder
killer.setKilled(true)
f.test(0)
// should have been removed
stores = f.Stores()
if len(stores) > 0 {
t.Fatalf("Finder did not remove dead store")
}
// but should stay in the DB
err = db.RunReadTx(func(ctx kvl.Ctx) error {
layer, err := meta.Open(ctx)
if err != nil {
return err
}
loc, err := layer.GetLocation(mock.UUID())
if err != nil {
return err
}
if loc == nil {
return fmt.Errorf("No location in database")
}
return nil
})
if err != nil {
t.Fatalf("Couldn't verify locations: %v", err)
}
// when the store comes back
killer.setKilled(false)
err = f.Rescan()
if err != nil {
t.Fatalf("Couldn't Rescan: %v", err)
}
// it should be there again
stores = f.Stores()
if _, ok := stores[mock.UUID()]; !ok {
t.Fatalf("Finder did not find uuid of directory store after resurrection")
}
}
