func getFullLruCache(t *testing.T) *TieredLRUCache {
cz := getCacheZone()
lru := New(cz, mockRemove, mock.NewLogger())
fillCache(t, lru)
return lru
}
func getFullLruCache(t *testing.T) *TieredLRUCache {
cz := getCacheZone()
lru := New(cz, mockRemove, mock.NewLogger())
storateObjects := (cz.StorageObjects / uint64(cacheTiers)) * uint64(cacheTiers)
for i := uint64(0); i < storateObjects; i++ {
oi := &types.ObjectIndex{
Part: uint32(i),
ObjID: types.NewObjectID("1.1", "/path/to/many/objects"),
}
for k := 0; k < cacheTiers; k++ {
lru.PromoteObject(oi)
}
}
if objects := lru.Stats().Objects(); objects != storateObjects {
t.Errorf("The cache was not full. Expected %d objects but it had %d",
storateObjects, objects)
}
return lru
}
func TestConstructor(t *testing.T) {
t.Parallel()
workingDiskPath, cleanup := testutils.GetTestFolder(t)
defer cleanup()
cfg := &config.CacheZone{Path: workingDiskPath, PartSize: 10}
l := mock.NewLogger()
if _, err := New(nil, l); err == nil {
t.Error("Expected to receive error with nil config")
}
if _, err := New(cfg, nil); err == nil {
t.Error("Expected to receive error with nil logger")
}
if _, err := New(&config.CacheZone{Path: "/an/invalid/path", PartSize: 10}, l); err == nil {
t.Error("Expected to receive error with an invalid path")
}
if _, err := New(&config.CacheZone{Path: "/", PartSize: 10}, l); err == nil {
t.Error("Expected to receive error with root path")
}
if _, err := New(&config.CacheZone{Path: workingDiskPath, PartSize: 0}, l); err == nil {
t.Error("Expected to receive error with invalid part size")
}
if _, err := New(cfg, l); err != nil {
t.Errorf("Received unexpected error while creating a normal disk storage: %s", err)
}
}
func testSetupWithStorage(t *testing.T, st types.Storage) (context.Context, *Handler, *types.Location) {
var cz = &types.CacheZone{
ID: "testZoen",
Algorithm: mock.NewCacheAlgorithm(&mock.CacheAlgorithmRepliers{
Remove: removeFunctionMock(t),
}),
Storage: st,
}
loc1 := &types.Location{
Logger: mock.NewLogger(),
Cache: cz,
CacheKey: cacheKey1,
Name: "location1",
}
loc2 := &types.Location{
Logger: mock.NewLogger(),
Cache: cz,
CacheKey: cacheKey2,
Name: "location2",
}
app := &mockApp{
getLocationFor: func(host, path string) *types.Location {
if host == host1 {
return loc1
}
if host == host2 {
return loc2
}
return nil
},
}
loc3 := &types.Location{
Logger: mock.NewLogger(),
}
ctx := contexts.NewAppContext(context.Background(), app)
purger, err := New(&config.Handler{}, loc3, nil)
if err != nil {
t.Fatal(err)
}
return ctx, purger, loc3
}
func TestSimpleUpstreamHeaders(t *testing.T) {
t.Parallel()
req, err := http.NewRequest("GET", "http://www.somewhere.com/err", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("test", "mest")
req.Header.Set("User-Agent", "nedomi") // The only exception...
headersCopy := make(http.Header)
for k, v := range req.Header {
headersCopy[k] = v
}
responded := false
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !reflect.DeepEqual(headersCopy, r.Header) {
t.Errorf("Different request headers: expected %#v, received %#v", headersCopy, r.Header)
}
responded = true
fmt.Fprint(w, "boo")
}))
defer ts.Close()
upstreamURL, err := url.Parse(ts.URL)
if err != nil {
t.Fatal(err)
}
up, err := upstream.NewSimple(upstreamURL)
if err != nil {
t.Fatal(err)
}
proxy, err := New(&config.Handler{}, &types.Location{
Name: "test",
Logger: mock.NewLogger(),
Upstream: up,
}, nil)
if err != nil {
t.Fatal(err)
}
resp := httptest.NewRecorder()
proxy.ServeHTTP(nil, resp, req)
if !responded {
t.Errorf("Server did not respond")
}
if resp.Body.String() != "boo" {
t.Errorf("Unexpected response %s", resp.Body)
}
}
func TestSimpleUpstream(t *testing.T) {
t.Parallel()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/err" {
w.WriteHeader(404)
fmt.Fprint(w, "error!")
return
}
w.WriteHeader(200)
fmt.Fprint(w, "hello world")
}))
defer ts.Close()
upstreamURL, err := url.Parse(ts.URL)
if err != nil {
t.Fatal(err)
}
up, err := upstream.NewSimple(upstreamURL)
if err != nil {
t.Fatal(err)
}
proxy, err := New(&config.Handler{}, &types.Location{
Name: "test",
Logger: mock.NewLogger(),
Upstream: up,
}, nil)
if err != nil {
t.Fatal(err)
}
req1, err := http.NewRequest("GET", "http://www.somewhere.com/err", nil)
if err != nil {
t.Fatal(err)
}
resp1 := httptest.NewRecorder()
proxy.ServeHTTP(nil, resp1, req1)
if resp1.Code != 404 || resp1.Body.String() != "error!" {
t.Errorf("Unexpected response %#v", resp1)
}
req2, err := http.NewRequest("GET", "http://www.somewhere.com/index", nil)
if err != nil {
t.Fatal(err)
}
resp2 := httptest.NewRecorder()
proxy.ServeHTTP(nil, resp2, req2)
if resp2.Code != 200 || resp2.Body.String() != "hello world" {
t.Errorf("Unexpected response %#v", resp2)
}
}
func TestCreatingBogusCacheAlgorithmReturnsError(t *testing.T) {
t.Parallel()
cz := config.CacheZone{
ID: "default",
Path: "/does/not/matter",
PartSize: 4123123,
StorageObjects: 9813743,
Algorithm: "bogus",
}
if _, err := New(&cz, mockRemove, mock.NewLogger()); err == nil {
t.Error("Expected an error when creating bogus algorithm but got none")
}
}
func TestCreatingCacheAlgorithms(t *testing.T) {
t.Parallel()
cz := config.CacheZone{
ID: "default",
Path: "/does/not/matter",
PartSize: 4123123,
StorageObjects: 9813743,
Algorithm: "lru",
}
if _, err := New(&cz, mockRemove, mock.NewLogger()); err != nil {
t.Errorf("Error when creating cache algorithm. %s", err)
}
}
func getTestDiskStorage(t *testing.T, partSize int) (*Disk, string, func()) {
diskPath, cleanup := testutils.GetTestFolder(t)
d, err := New(&config.CacheZone{
Path: diskPath,
PartSize: types.BytesSize(partSize),
}, mock.NewLogger())
if err != nil {
t.Fatalf("Could not create storage: %s", err)
}
return d, diskPath, cleanup
}
func TestShouldKeepMethod(t *testing.T) {
t.Parallel()
cz := getCacheZone()
oi := getObjectIndex()
lru := New(cz, nil, mock.NewLogger())
if shouldKeep := lru.ShouldKeep(oi); !shouldKeep {
t.Error("LRU cache was supposed to return true for all ShouldKeep questions" +
"but it returned false")
}
if objects := lru.Stats().Objects(); objects != 1 {
t.Error("ShouldKeep was suppsed to add the object into the cache but id did not")
}
if shouldKeep := lru.ShouldKeep(oi); !shouldKeep {
t.Error("ShouldKeep returned false after its second call")
}
}
func TestPromotionsInEmptyCache(t *testing.T) {
t.Parallel()
cz := getCacheZone()
oi := getObjectIndex()
lru := New(cz, nil, mock.NewLogger())
lru.PromoteObject(oi)
if objects := lru.Stats().Objects(); objects != 1 {
t.Errorf("Expected 1 object but found %d", objects)
}
lruEl, ok := lru.lookup[oi.Hash()]
if !ok {
t.Error("Was not able to find the object in the LRU table")
}
if lruEl.ListTier != cacheTiers-1 {
t.Errorf("Object was not in the last tier but in %d", lruEl.ListTier)
}
lru.PromoteObject(oi)
if lruEl.ListTier != cacheTiers-2 {
t.Errorf("Promoted object did not change its tier. "+
"Expected it to be at tier %d but it was at %d", cacheTiers-2,
lruEl.ListTier)
}
for i := 0; i < cacheTiers; i++ {
lru.PromoteObject(oi)
}
if lruEl.ListTier != 0 {
t.Errorf("Expected the promoted object to be in the uppermost "+
"tier but it was at tier %d", lruEl.ListTier)
}
}
func TestSize(t *testing.T) {
t.Parallel()
cz := getCacheZone()
oi := getObjectIndex()
lru := New(cz, nil, mock.NewLogger())
if err := lru.AddObject(oi); err != nil {
t.Errorf("Error adding object into the cache. %s", err)
}
if objects := lru.Stats().Objects(); objects != 1 {
t.Errorf("Expec 1 object but found %d", objects)
}
if err := lru.AddObject(oi); err == nil {
t.Error("Exepected error when adding object for the second time")
}
for i := 0; i < 16; i++ {
oii := &types.ObjectIndex{
Part: uint32(i),
ObjID: types.NewObjectID("1.1", "/path/to/other/object"),
}
if err := lru.AddObject(oii); err != nil {
t.Errorf("Adding object in cache. %s", err)
}
}
if objects := lru.Stats().Objects(); objects != 17 {
t.Errorf("Expec 17 objects but found %d", objects)
}
if size, expected := lru.ConsumedSize(), 17*cz.PartSize; size != expected {
t.Errorf("Expected total size to be %d but it was %d", expected, size)
}
}
func TestDiskReload(t *testing.T) {
t.Parallel()
tempDir, cleanup := testutils.GetTestFolder(t)
defer cleanup()
app, err := New(types.AppVersion{}, getConfigGetter(tempDir))
if err != nil {
t.Fatalf("Could not create an application: %s", err)
}
stor, err := disk.New(app.cfg.CacheZones["default"], mock.NewLogger())
if err != nil {
t.Fatalf("Could not initialize a storage: %s", err)
}
objIDNew := types.NewObjectID("key", "new")
objIDOld := types.NewObjectID("key", "old")
testutils.ShouldntFail(t,
stor.SaveMetadata(&types.ObjectMetadata{ID: objIDNew, ExpiresAt: time.Now().Unix() + 600}),
stor.SaveMetadata(&types.ObjectMetadata{ID: objIDOld, ExpiresAt: time.Now().Unix() - 600}),
stor.SavePart(&types.ObjectIndex{ObjID: objIDNew, Part: 0}, strings.NewReader("test1-1")),
stor.SavePart(&types.ObjectIndex{ObjID: objIDNew, Part: 1}, strings.NewReader("test1-2")),
stor.SavePart(&types.ObjectIndex{ObjID: objIDOld, Part: 0}, strings.NewReader("test2-1")),
)
if err := app.initFromConfig(); err != nil {
t.Fatalf("Could not init from config: %s", err)
}
defer app.ctxCancel()
time.Sleep(1 * time.Second)
const expectedObjects = 2
cacheObjects := app.cacheZones["default"].Algorithm.Stats().Objects()
if cacheObjects != expectedObjects {
t.Errorf("Expected object count in cache to be %d but it was %d", expectedObjects, cacheObjects)
}
}
func TestLookupAndRemove(t *testing.T) {
t.Parallel()
cz := getCacheZone()
oi := getObjectIndex()
lru := New(cz, nil, mock.NewLogger())
if lru.Lookup(oi) {
t.Error("Empty LRU cache returned True for a object index lookup")
}
if err := lru.AddObject(oi); err != nil {
t.Errorf("Error adding object into the cache. %s", err)
}
oi = getObjectIndex() // get a new/same objectIndex
if !lru.Lookup(oi) {
t.Error("Lookup for object index which was just added returned false")
}
lru.Remove(oi)
if lru.Lookup(oi) {
t.Error("Lookup for object index which was just removed returned true")
}
}
func TestCreatingCacheAlgorithmWithNilConfigReturnsError(t *testing.T) {
t.Parallel()
if _, err := New(nil, mockRemove, mock.NewLogger()); err == nil {
t.Error("Expected an error when creating bogus algorithm but got none")
}
}
func TestSimpleRetryWithNilUpstream(t *testing.T) {
t.Parallel()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/err" {
w.WriteHeader(404)
fmt.Fprint(w, "error!")
return
}
w.WriteHeader(200)
fmt.Fprint(w, "hello world")
}))
defer ts.Close()
upstreamURL, err := url.Parse(ts.URL)
if err != nil {
t.Fatal(err)
}
up, err := upstream.NewSimple(upstreamURL)
if err != nil {
t.Fatal(err)
}
retryTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/err" {
w.WriteHeader(200)
fmt.Fprint(w, "not error!")
return
}
w.WriteHeader(404)
fmt.Fprint(w, "not hello world")
}))
defer retryTs.Close()
retryUpstreamURL, err := url.Parse(retryTs.URL)
if err != nil {
t.Fatal(err)
}
retryUpstream, err := upstream.NewSimple(retryUpstreamURL)
if err != nil {
t.Fatal(err)
}
proxy, err := New(
config.NewHandler("proxy", json.RawMessage(`{ "try_other_upstream_on_code" : {"404": "nonexistant_upstream"}}`)),
&types.Location{
Name: "test",
Logger: mock.NewLogger(),
Upstream: up,
}, nil)
if err != nil {
t.Fatal(err)
}
req1, err := http.NewRequest("GET", "http://www.somewhere.com/err", nil)
if err != nil {
t.Fatal(err)
}
resp1 := httptest.NewRecorder()
ctx := contexts.NewAppContext(context.Background(), &mockApp{
upstreams: map[string]types.Upstream{
"retry_upstream": retryUpstream,
},
})
proxy.ServeHTTP(ctx, resp1, req1)
if resp1.Code != 404 || resp1.Body.String() != "error!" {
t.Errorf("Unexpected response %#v", resp1)
}
req2, err := http.NewRequest("GET", "http://www.somewhere.com/index", nil)
if err != nil {
t.Fatal(err)
}
resp2 := httptest.NewRecorder()
proxy.ServeHTTP(nil, resp2, req2)
if resp2.Code != 200 || resp2.Body.String() != "hello world" {
t.Errorf("Unexpected response %#v", resp2)
}
}