func generateFiles(n int) map[string]string {
var files = make(map[string]string, n)
for i := 0; n > i; i++ {
name := testutils.GenerateMeAString(int64(i), 5)
files[name] = testutils.GenerateMeAString(int64(i*n), rand.Int63n(500)+200)
}
return files
}
func TestWeightedAlgorithms(t *testing.T) {
t.Parallel()
wg := sync.WaitGroup{}
testAlgorithmForWeight := func(id string, inst types.UpstreamBalancingAlgorithm) {
isWeighted := !strings.HasPrefix(id, unweightedPrefix)
upstreams := testutils.GetRandomUpstreams(5, 20)
inst.Set(upstreams)
var totalWeight uint32
weights := map[string]uint32{}
for _, u := range upstreams {
weights[u.Host] = u.Weight
totalWeight += u.Weight
}
mapping := map[string]uint32{}
urlsToTest := uint32(25000)
for i := uint32(0); i < urlsToTest; i++ {
url := testutils.GenerateMeAString(rand.Int63(), 5+rand.Int63n(100))
if res, err := inst.Get(url); err != nil {
t.Errorf("Unexpected error when getting url %s from algorithm %s: %s", url, id, err)
} else {
mapping[res.Host]++
}
}
var deviation float64
expectedPercent := 100.0 / float64(len(upstreams))
for host, count := range mapping {
if isWeighted {
expectedPercent = float64(weights[host]) * 100.0 / float64(totalWeight)
}
actualPercent := float64(count) * 100 / float64(urlsToTest)
deviation += math.Abs(actualPercent - expectedPercent)
t.Logf("Algorithm %s, host %s: expected percent %f (weight %d out of %d) but got %f (%d out of %d); deviation %f%%\n",
id, host, expectedPercent, weights[host], totalWeight, actualPercent, count, urlsToTest, math.Abs(actualPercent-expectedPercent))
}
threshold := 3.0
avgDeviation := deviation / float64(len(upstreams))
if avgDeviation > threshold {
t.Errorf("The average deviation for algorithm %s is above the threshold of %f%%: %f",
id, threshold, avgDeviation)
}
wg.Done()
}
for id, constructor := range allAlgorithms {
wg.Add(1)
go testAlgorithmForWeight(id, constructor())
}
wg.Wait()
}
func Test2PartsFile(t *testing.T) {
var fsmap = make(map[string]string)
var file = "2parts"
fsmap[file] = testutils.GenerateMeAString(2, 10)
t.Parallel()
app := newTestAppFromMap(t, fsmap)
defer app.cleanup()
app.testRange(file, 2, 8)
app.testFullRequest(file)
}
func consistenHashAlgorithmTest(t *testing.T, wg *sync.WaitGroup, id string) {
inst := allAlgorithms[id]()
upstreams := testutils.GetRandomUpstreams(10, 100)
inst.Set(upstreams)
urlsToTest := 3000 + rand.Intn(1000)
mapping := map[string]string{}
for i := 0; i < urlsToTest; i++ {
url := testutils.GenerateMeAString(rand.Int63(), 5+rand.Int63n(100))
if res1, err := inst.Get(url); err != nil {
t.Errorf("Unexpected error when getting url %s from algorithm %s: %s", url, id, err)
} else if res2, err := inst.Get(url); err != nil {
t.Errorf("Unexpected error when getting url %s from algorithm %s for the second time: %s", url, id, err)
} else if !reflect.DeepEqual(res1, res2) {
t.Errorf("The two results for url %s by algorithm %s are different: %#v and %#v", url, id, res1, res2)
} else {
mapping[url] = res1.Host
}
}
oldWeght := getTotalWeight(upstreams)
checkDeviation := func(newUpstreams []*types.UpstreamAddress) {
inst.Set(newUpstreams)
var sameCount float64
for url, oldHost := range mapping {
if res, err := inst.Get(url); err != nil {
t.Errorf("Unexpected error when getting url %s from algorithm %s: %s", url, id, err)
} else if res.Host == oldHost {
sameCount++
}
}
total := float64(len(mapping))
newWeght := getTotalWeight(newUpstreams)
weightDiff := math.Abs(oldWeght - newWeght)
if math.Abs(weightDiff/oldWeght-(total-sameCount)/total) > 0.25 {
t.Errorf("[%s] Same count is %f of %f (count diff %f%%); upstreams are %d out of %d (weight diff %f-%f=%f or %f%%); deviation from expected: %f\n\n",
id, sameCount, total, (total-sameCount)*100/total,
len(newUpstreams), len(upstreams), oldWeght, newWeght, oldWeght-newWeght, weightDiff*100/oldWeght,
math.Abs(weightDiff/oldWeght-(total-sameCount)/total)*100)
}
}
// Add an extra server at the start
newUpstream := testutils.GetUpstream(len(upstreams))
checkDeviation(append([]*types.UpstreamAddress{newUpstream}, upstreams...))
// Remove a random server
randomServer := rand.Intn(len(upstreams))
checkDeviation(append(upstreams[:randomServer], upstreams[randomServer+1:]...))
// Add an extra server at that point
checkDeviation(append(upstreams[:randomServer], append([]*types.UpstreamAddress{newUpstream}, upstreams[randomServer:]...)...))
// Add an extra server at the end
checkDeviation(append(upstreams, newUpstream))
wg.Done()
}
func runTest(b *testing.B, id string) {
b.ReportAllocs()
inst := allAlgorithms[id]()
inst.Set(testutils.GetUpstreams(1, upstremsCount))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if _, err := inst.Get(testutils.GenerateMeAString(0, urlLength)); err != nil {
b.Fatal(err)
}
}
})
}
// Tests the storage headers map in multithreading usage. An error will be
// triggered by a race condition. This may or may not happen. So no failures
// in the test do not mean there are no problems. But it may catch an error
// from time to time. In fact it does quite often for me.
//
// Most of the time the test fails with a panic. And most of the time
// the panic is in the runtime. So isntead of a error message via t.Error
// the test fails with a panic.
func TestStorageHeadersFunctionWithManyGoroutines(t *testing.T) {
t.Parallel()
app := newTestApp(t)
defer app.cleanup()
headerKeyFunc := func(i int) string {
return fmt.Sprintf("X-Header-%d", i)
}
headerValueFunc := func(i int) string {
return fmt.Sprintf("value-%d", i)
}
var pathFile = testutils.GenerateMeAString(20, 30)
// setup the response
for i := 0; i < 100; i++ {
handler := func(num int) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Add(headerKeyFunc(num), headerValueFunc(num))
fmt.Fprintf(w, pathFile)
}
}(i)
app.up.Handle("/path/"+strconv.Itoa(i), http.HandlerFunc(handler))
}
testFunc := func(t *testing.T, i, j int) {
rec := httptest.NewRecorder()
req, err := http.NewRequest("HEAD", "/path/"+strconv.Itoa(i), nil)
if err != nil {
t.Fatal(err)
}
app.cacheHandler.RequestHandle(context.Background(), rec, req)
val := rec.Header().Get(headerKeyFunc(i))
expVal := headerValueFunc(i)
if val != expVal {
t.Errorf("Expected header value %s and received %s", expVal, val)
}
}
concurrentTestHelper(t, 20, 100, testFunc)
}
func TestVeryFragmentedFile(t *testing.T) {
t.Parallel()
app := newTestApp(t)
var file = "long"
app.fsmap[file] = testutils.GenerateMeAString(1, 2000)
defer app.cleanup()
app.testRange(file, 5, 10)
app.testRange(file, 5, 2)
app.testRange(file, 2, 2)
app.testRange(file, 20, 10)
app.testRange(file, 30, 10)
app.testRange(file, 40, 10)
app.testRange(file, 50, 10)
app.testRange(file, 60, 10)
app.testRange(file, 70, 10)
app.testRange(file, 50, 20)
app.testRange(file, 200, 5)
app.testFullRequest(file)
app.testRange(file, 3, 1000)
}
for pb.Next() {
lru := aFullCache(b, benchCacheSize)
if lru.stats().Objects() != benchCacheSize {
b.Fatalf("expected %d objects in the full lru but got %d",
benchCacheSize,
lru.stats().Objects())
}
}
})
}
var randPath = func() func() string {
ch := make(chan string, 1000)
go func() {
for seed := int64(0); ; seed++ {
ch <- testutils.GenerateMeAString(seed, 20)
}
}()
return func() string {
return <-ch
}
}()
func BenchmarkLookupAndRemove(b *testing.B) {
b.StopTimer()
lru := aFullCache(b, benchCacheSize)
var tooFast uint64
b.StartTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
oi := getObjectIndexFor(randUint32(), "1.1", randPath())