func TestDiskPaths(t *testing.T) {
t.Parallel()
idx := &types.ObjectIndex{
ObjID: types.NewObjectID("1.2", "/somewhere2"),
Part: 33,
}
diskPath := "/some/path"
disk := &Disk{path: diskPath}
hash := idx.ObjID.StrHash()
expectedHash := "052fb8b15a7737b1e7b70546b1c5023f0bd00a7d"
if hash != expectedHash {
t.Errorf("Incorrect ObjectID hash. Exected %s, got %s", expectedHash, hash)
}
objIDPath := disk.getObjectIDPath(idx.ObjID)
expectedObjIDPath := filepath.Join(diskPath, idx.ObjID.CacheKey(), expectedHash[:2], expectedHash[2:4], expectedHash)
if objIDPath != expectedObjIDPath {
t.Errorf("Incorrect ObjectID path. Exected %s, got %s", expectedObjIDPath, objIDPath)
}
objIdxPath := disk.getObjectIndexPath(idx)
expectedObjIdxPath := filepath.Join(expectedObjIDPath, "000033")
if objIdxPath != expectedObjIdxPath {
t.Errorf("Incorrect ObjectIndex path. Exected %s, got %s", expectedObjIdxPath, objIdxPath)
}
objMetadataPath := disk.getObjectMetadataPath(idx.ObjID)
expectedObjMetadataPath := filepath.Join(expectedObjIDPath, objectMetadataFileName)
if objMetadataPath != expectedObjMetadataPath {
t.Errorf("Incorrect ObjectMetadata path. Exected %s, got %s", expectedObjMetadataPath, objMetadataPath)
}
}
func getFullLruCache(t *testing.T) *TieredLRUCache {
cz := getCacheZone()
lru := New(cz, mockRemove, mock.NewLogger())
storateObjects := (cz.StorageObjects / uint64(cacheTiers)) * uint64(cacheTiers)
for i := uint64(0); i < storateObjects; i++ {
oi := &types.ObjectIndex{
Part: uint32(i),
ObjID: types.NewObjectID("1.1", "/path/to/many/objects"),
}
for k := 0; k < cacheTiers; k++ {
lru.PromoteObject(oi)
}
}
if objects := lru.Stats().Objects(); objects != storateObjects {
t.Errorf("The cache was not full. Expected %d objects but it had %d",
storateObjects, objects)
}
return lru
}
func TestPromotionToTheFrontOfTheList(t *testing.T) {
t.Parallel()
lru := getFullLruCache(t)
testOiFirst := &types.ObjectIndex{
Part: 0,
ObjID: types.NewObjectID("1.1", "/path/to/tested/object"),
}
testOiSecond := &types.ObjectIndex{
Part: 1,
ObjID: types.NewObjectID("1.1", "/path/to/tested/object"),
}
for currentTier := cacheTiers - 1; currentTier >= 0; currentTier-- {
lru.PromoteObject(testOiFirst)
lru.PromoteObject(testOiSecond)
}
// First promoting the first object to the front of the list
lru.PromoteObject(testOiFirst)
lruEl, ok := lru.lookup[testOiFirst.Hash()]
if !ok {
t.Fatal("Recently added object was not in the lookup table")
}
if lru.tiers[0].Front() != lruEl.ListElem {
t.Error("The expected element was not at the front of the top list")
}
// Then promoting the second one
lru.PromoteObject(testOiSecond)
lruEl, ok = lru.lookup[testOiSecond.Hash()]
if !ok {
t.Fatal("Recently added object was not in the lookup table")
}
if lru.tiers[0].Front() != lruEl.ListElem {
t.Error("The expected element was not at the front of the top list")
}
}
func TestDiskReload(t *testing.T) {
t.Parallel()
tempDir, cleanup := testutils.GetTestFolder(t)
defer cleanup()
app, err := New(types.AppVersion{}, getConfigGetter(tempDir))
if err != nil {
t.Fatalf("Could not create an application: %s", err)
}
stor, err := disk.New(app.cfg.CacheZones["default"], mock.NewLogger())
if err != nil {
t.Fatalf("Could not initialize a storage: %s", err)
}
objIDNew := types.NewObjectID("key", "new")
objIDOld := types.NewObjectID("key", "old")
testutils.ShouldntFail(t,
stor.SaveMetadata(&types.ObjectMetadata{ID: objIDNew, ExpiresAt: time.Now().Unix() + 600}),
stor.SaveMetadata(&types.ObjectMetadata{ID: objIDOld, ExpiresAt: time.Now().Unix() - 600}),
stor.SavePart(&types.ObjectIndex{ObjID: objIDNew, Part: 0}, strings.NewReader("test1-1")),
stor.SavePart(&types.ObjectIndex{ObjID: objIDNew, Part: 1}, strings.NewReader("test1-2")),
stor.SavePart(&types.ObjectIndex{ObjID: objIDOld, Part: 0}, strings.NewReader("test2-1")),
)
if err := app.initFromConfig(); err != nil {
t.Fatalf("Could not init from config: %s", err)
}
defer app.ctxCancel()
time.Sleep(1 * time.Second)
const expectedObjects = 2
cacheObjects := app.cacheZones["default"].Algorithm.Stats().Objects()
if cacheObjects != expectedObjects {
t.Errorf("Expected object count in cache to be %d but it was %d", expectedObjects, cacheObjects)
}
}
func TestResizeUp(t *testing.T) {
t.Parallel()
lru := getFullLruCache(t)
testOi := &types.ObjectIndex{
Part: 0,
ObjID: types.NewObjectID("1.1", "/path/to/tested/object"),
}
oldSize := lru.Stats().Objects()
lru.ChangeConfig(10, 50, oldSize+20, lru.logger)
lru.PromoteObject(testOi)
if lru.Stats().Objects() != oldSize+1 {
t.Errorf("It was expected that after resize more objects could be added but that wasn't true")
}
}
func TestBreakInIndexes(t *testing.T) {
t.Parallel()
id := types.NewObjectID("test", "mest")
for index, test := range breakInIndexesMatrix {
var result = BreakInIndexes(id, test.start, test.end, test.partSize)
if len(result) != len(test.result) {
t.Errorf("Wrong len (%d != %d) on test index %d", len(result), len(test.result), index)
}
for resultIndex, value := range result {
if value.Part != test.result[resultIndex] {
t.Errorf("Wrong part for test index %d, wanted %d in position %d but got %d", index, test.result[resultIndex], resultIndex, value.Part)
}
}
}
}
func fillCache(t testing.TB, lru *TieredLRUCache) {
storateObjects := (lru.cfg.StorageObjects / uint64(cacheTiers)) * uint64(cacheTiers)
for i := uint64(0); i < storateObjects; i++ {
oi := &types.ObjectIndex{
Part: uint32(i),
ObjID: types.NewObjectID("1.1", "/path/to/many/objects"),
}
for k := uint64(0); k < cacheTiers-(i/(storateObjects/cacheTiers)); k++ {
lru.PromoteObject(oi)
}
}
if objects := lru.Stats().Objects(); objects != storateObjects {
t.Errorf("The cache was not full. Expected %d objects but it had %d",
storateObjects, objects)
}
}
func TestResizeDown(t *testing.T) {
t.Parallel()
lru := getFullLruCache(t)
oldSize := lru.Stats().Objects()
lru.ChangeConfig(1, 1, oldSize/2, lru.logger)
var ch = make(chan struct{})
var wg sync.WaitGroup
for i := 0; 30 > i; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
oi := &types.ObjectIndex{Part: uint32(i), ObjID: types.NewObjectID("1.1", "/path/to/some")}
timer := time.NewTimer(time.Millisecond)
for {
select {
case <-ch:
return
default:
timer.Reset(time.Millisecond)
<-timer.C
lru.PromoteObject(oi)
}
}
}(i)
}
var testSize = func() {
if oldSize/2 < lru.Stats().Objects() {
t.Errorf("It was expected that after resize down the size will be less than or equal to %d but it's %d",
oldSize/2, lru.Stats().Objects())
}
}
testSize()
time.Sleep(50 * time.Millisecond) // give time for the Resize down to remove objects
close(ch)
wg.Wait()
testSize()
}
func TestPromotionInFullCache(t *testing.T) {
t.Parallel()
lru := getFullLruCache(t)
testOi := &types.ObjectIndex{
Part: 0,
ObjID: types.NewObjectID("1.1", "/path/to/tested/object"),
}
for currentTier := cacheTiers - 1; currentTier >= 0; currentTier-- {
lru.PromoteObject(testOi)
lruEl, ok := lru.lookup[testOi.Hash()]
if !ok {
t.Fatalf("Lost object while promoting it to tier %d", currentTier)
}
if lruEl.ListTier != currentTier {
t.Errorf("Tested LRU was not in the expected tier. It was suppsed to be"+
" in tier %d but it was in %d", currentTier, lruEl.ListTier)
}
}
}
func TestSize(t *testing.T) {
t.Parallel()
cz := getCacheZone()
oi := getObjectIndex()
lru := New(cz, nil, mock.NewLogger())
if err := lru.AddObject(oi); err != nil {
t.Errorf("Error adding object into the cache. %s", err)
}
if objects := lru.Stats().Objects(); objects != 1 {
t.Errorf("Expec 1 object but found %d", objects)
}
if err := lru.AddObject(oi); err == nil {
t.Error("Exepected error when adding object for the second time")
}
for i := 0; i < 16; i++ {
oii := &types.ObjectIndex{
Part: uint32(i),
ObjID: types.NewObjectID("1.1", "/path/to/other/object"),
}
if err := lru.AddObject(oii); err != nil {
t.Errorf("Adding object in cache. %s", err)
}
}
if objects := lru.Stats().Objects(); objects != 17 {
t.Errorf("Expec 17 objects but found %d", objects)
}
if size, expected := lru.ConsumedSize(), 17*cz.PartSize; size != expected {
t.Errorf("Expected total size to be %d but it was %d", expected, size)
}
}
func getObjectIndex() *types.ObjectIndex {
return &types.ObjectIndex{
Part: 3,
ObjID: types.NewObjectID("1.1", "/path"),
}
}
package mock
import (
"errors"
"testing"
"github.com/ironsmile/nedomi/types"
)
var idx = &types.ObjectIndex{
ObjID: types.NewObjectID("test", "/path/to/object"),
Part: 14,
}
var _ types.CacheAlgorithm = new(CacheAlgorithm) // make sure the mock implements the interface
func TestMockCacheAlgorithm(t *testing.T) {
t.Parallel()
d := NewCacheAlgorithm(nil)
if d.Defaults.Lookup(idx) || d.Defaults.ShouldKeep(idx) || d.Defaults.AddObject(idx) != nil {
t.Errorf("Invalid default default replies %#v", d.Defaults)
}
c1 := NewCacheAlgorithm(&CacheAlgorithmRepliers{
Lookup: func(*types.ObjectIndex) bool { return true },
})
if !c1.Defaults.Lookup(idx) || c1.Defaults.ShouldKeep(idx) || c1.Defaults.AddObject(idx) != nil {
t.Error("Invalid partial custom default replies")
}
promoted := false
func getObjectIndexFor(part uint32, key, path string) *types.ObjectIndex {
return &types.ObjectIndex{
ObjID: types.NewObjectID(key, path),
Part: part,
}
}
"os"
"testing"
"time"
"github.com/ironsmile/nedomi/mock"
"github.com/ironsmile/nedomi/types"
"github.com/ironsmile/nedomi/utils"
"github.com/ironsmile/nedomi/utils/httputils"
)
const (
input = "The input of the test"
inputSize = uint64(len(input))
)
var oid = types.NewObjectID("testKey1", "testPathA")
var oMeta = &types.ObjectMetadata{
ID: oid,
ResponseTimestamp: time.Now().Unix(),
Code: 200,
Size: inputSize,
Headers: nil,
}
func TestPartWriter(t *testing.T) {
t.Parallel()
test := func(start, length, partSize uint64) {
cz := &types.CacheZone{
ID: "TestCZ",
Storage: mock.NewStorage(partSize),
"sync"
"testing"
"time"
"github.com/ironsmile/nedomi/config"
"github.com/ironsmile/nedomi/mock"
"github.com/ironsmile/nedomi/types"
"github.com/ironsmile/nedomi/utils/testutils"
)
func init() {
rand.Seed(time.Now().Unix())
}
var obj1 = &types.ObjectMetadata{
ID: types.NewObjectID("testkey", "/lorem/ipsum"),
ResponseTimestamp: time.Now().Unix(),
Headers: http.Header{"test": []string{"mest"}},
}
var obj2 = &types.ObjectMetadata{
ID: types.NewObjectID("concern", "/doge?so=scare&very_parameters"),
ResponseTimestamp: time.Now().Unix(),
Headers: http.Header{"how-to": []string{"header"}},
}
var obj3 = &types.ObjectMetadata{
ID: types.NewObjectID("concern", "/very/space**"),
ResponseTimestamp: time.Now().Unix(),
Headers: http.Header{"so": []string{"galaxy", "amaze"}},
}
func checkFile(t *testing.T, d *Disk, filePath, expectedContents string) {
const (
cacheKey1, cacheKey2 = "testkey1", "testkey2"
testURL = "http://example.com/more/path*!:@#>"
host1, host2, host3 = "example.org:1232", "example.net:8080", "notexample.net:8080"
path1, path2, path3 = "/path/to/object", "/path/to/an/object", "/path/to/no/object"
url1, url2, url3 = "http://" + host1 + path1, "http://" + host2 + path2, "http://" + host3 + path3
host4, host5, host6 = "example.org:1232", "example.org:notPOrt", "notexample.net:8080"
path4, path5, path6 = "/path/to/no/object", "/path/to/object", "/path/to/no/object"
url4, url5, url6 = "http://" + host4 + path4, "http://" + host5 + path5, "http://" + host6 + path6
requestText = `[ "` + url1 + `", "` + url2 + `", "` + url3 + `", "` + url4 + `", "` + url5 + `", "` + url6 + `" ]`
badRequestText = `Bad request`
)
var (
obj1 = types.NewObjectID(cacheKey1, path1)
obj2 = types.NewObjectID(cacheKey2, path2)
)
func testCode(t *testing.T, code, expected int) {
if code != expected {
t.Fatalf("wrong response code %d expected %d", code, expected)
}
}
func removeFunctionMock(t *testing.T) func(parts ...*types.ObjectIndex) {
return func(parts ...*types.ObjectIndex) {
for _, part := range parts {
switch part.ObjID.Path() {
case obj1.Path(), obj2.Path():
default:
"io/ioutil"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/ironsmile/nedomi/types"
"github.com/ironsmile/nedomi/utils/testutils"
)
// make sure that NewStorage implements types.Storage
var _ types.Storage = (*Storage)(nil)
var obj1 = &types.ObjectMetadata{
ID: types.NewObjectID("testkey", "/lorem/ipsum"),
ResponseTimestamp: time.Now().Unix(),
Headers: http.Header{"test": []string{"mest"}},
}
var obj2 = &types.ObjectMetadata{
ID: types.NewObjectID("testkey", "/lorem/ipsum/2"),
ResponseTimestamp: time.Now().Unix(),
Headers: http.Header{},
}
func TestMockStorageExpectedErrors(t *testing.T) {
t.Parallel()
s := NewStorage(10)
idx := &types.ObjectIndex{ObjID: obj1.ID, Part: 5}
