Exemple #1
0
func generateFiles(n int) map[string]string {
	var files = make(map[string]string, n)
	for i := 0; n > i; i++ {
		name := testutils.GenerateMeAString(int64(i), 5)
		files[name] = testutils.GenerateMeAString(int64(i*n), rand.Int63n(500)+200)
	}
	return files
}
func TestWeightedAlgorithms(t *testing.T) {
	t.Parallel()
	wg := sync.WaitGroup{}

	testAlgorithmForWeight := func(id string, inst types.UpstreamBalancingAlgorithm) {
		isWeighted := !strings.HasPrefix(id, unweightedPrefix)

		upstreams := testutils.GetRandomUpstreams(5, 20)
		inst.Set(upstreams)

		var totalWeight uint32
		weights := map[string]uint32{}
		for _, u := range upstreams {
			weights[u.Host] = u.Weight
			totalWeight += u.Weight
		}

		mapping := map[string]uint32{}
		urlsToTest := uint32(25000)
		for i := uint32(0); i < urlsToTest; i++ {
			url := testutils.GenerateMeAString(rand.Int63(), 5+rand.Int63n(100))
			if res, err := inst.Get(url); err != nil {
				t.Errorf("Unexpected error when getting url %s from algorithm %s: %s", url, id, err)
			} else {
				mapping[res.Host]++
			}
		}

		var deviation float64
		expectedPercent := 100.0 / float64(len(upstreams))
		for host, count := range mapping {
			if isWeighted {
				expectedPercent = float64(weights[host]) * 100.0 / float64(totalWeight)
			}

			actualPercent := float64(count) * 100 / float64(urlsToTest)
			deviation += math.Abs(actualPercent - expectedPercent)

			t.Logf("Algorithm %s, host %s: expected percent %f (weight %d out of %d) but got %f (%d out of %d); deviation %f%%\n",
				id, host, expectedPercent, weights[host], totalWeight, actualPercent, count, urlsToTest, math.Abs(actualPercent-expectedPercent))
		}

		threshold := 3.0
		avgDeviation := deviation / float64(len(upstreams))
		if avgDeviation > threshold {
			t.Errorf("The average deviation for algorithm %s is above the threshold of %f%%: %f",
				id, threshold, avgDeviation)
		}

		wg.Done()
	}

	for id, constructor := range allAlgorithms {
		wg.Add(1)
		go testAlgorithmForWeight(id, constructor())
	}

	wg.Wait()
}
Exemple #3
0
func Test2PartsFile(t *testing.T) {
	var fsmap = make(map[string]string)
	var file = "2parts"
	fsmap[file] = testutils.GenerateMeAString(2, 10)
	t.Parallel()
	app := newTestAppFromMap(t, fsmap)
	defer app.cleanup()
	app.testRange(file, 2, 8)
	app.testFullRequest(file)
}
func consistenHashAlgorithmTest(t *testing.T, wg *sync.WaitGroup, id string) {
	inst := allAlgorithms[id]()
	upstreams := testutils.GetRandomUpstreams(10, 100)
	inst.Set(upstreams)

	urlsToTest := 3000 + rand.Intn(1000)
	mapping := map[string]string{}
	for i := 0; i < urlsToTest; i++ {
		url := testutils.GenerateMeAString(rand.Int63(), 5+rand.Int63n(100))
		if res1, err := inst.Get(url); err != nil {
			t.Errorf("Unexpected error when getting url %s from algorithm %s: %s", url, id, err)
		} else if res2, err := inst.Get(url); err != nil {
			t.Errorf("Unexpected error when getting url %s from algorithm %s for the second time: %s", url, id, err)
		} else if !reflect.DeepEqual(res1, res2) {
			t.Errorf("The two results for url %s by algorithm %s are different: %#v and %#v", url, id, res1, res2)
		} else {
			mapping[url] = res1.Host
		}
	}

	oldWeght := getTotalWeight(upstreams)
	checkDeviation := func(newUpstreams []*types.UpstreamAddress) {
		inst.Set(newUpstreams)
		var sameCount float64
		for url, oldHost := range mapping {
			if res, err := inst.Get(url); err != nil {
				t.Errorf("Unexpected error when getting url %s from algorithm %s: %s", url, id, err)
			} else if res.Host == oldHost {
				sameCount++
			}
		}
		total := float64(len(mapping))
		newWeght := getTotalWeight(newUpstreams)
		weightDiff := math.Abs(oldWeght - newWeght)

		if math.Abs(weightDiff/oldWeght-(total-sameCount)/total) > 0.25 {
			t.Errorf("[%s] Same count is %f of %f (count diff %f%%); upstreams are %d out of %d (weight diff %f-%f=%f or %f%%); deviation from expected: %f\n\n",
				id, sameCount, total, (total-sameCount)*100/total,
				len(newUpstreams), len(upstreams), oldWeght, newWeght, oldWeght-newWeght, weightDiff*100/oldWeght,
				math.Abs(weightDiff/oldWeght-(total-sameCount)/total)*100)
		}
	}

	// Add an extra server at the start
	newUpstream := testutils.GetUpstream(len(upstreams))
	checkDeviation(append([]*types.UpstreamAddress{newUpstream}, upstreams...))
	// Remove a random server
	randomServer := rand.Intn(len(upstreams))
	checkDeviation(append(upstreams[:randomServer], upstreams[randomServer+1:]...))
	// Add an extra server at that point
	checkDeviation(append(upstreams[:randomServer], append([]*types.UpstreamAddress{newUpstream}, upstreams[randomServer:]...)...))
	// Add an extra server at the end
	checkDeviation(append(upstreams, newUpstream))
	wg.Done()
}
func runTest(b *testing.B, id string) {
	b.ReportAllocs()

	inst := allAlgorithms[id]()
	inst.Set(testutils.GetUpstreams(1, upstremsCount))

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			if _, err := inst.Get(testutils.GenerateMeAString(0, urlLength)); err != nil {
				b.Fatal(err)
			}
		}
	})
}
Exemple #6
0
// Tests the storage headers map in multithreading usage. An error will be
// triggered by a race condition. This may or may not happen. So no failures
// in the test do not mean there are no problems. But it may catch an error
// from time to time. In fact it does quite often for me.
//
// Most of the time the test fails with a panic. And most of the time
// the panic is in the runtime. So isntead of a error message via t.Error
// the test fails with a panic.
func TestStorageHeadersFunctionWithManyGoroutines(t *testing.T) {
	t.Parallel()
	app := newTestApp(t)
	defer app.cleanup()

	headerKeyFunc := func(i int) string {
		return fmt.Sprintf("X-Header-%d", i)
	}

	headerValueFunc := func(i int) string {
		return fmt.Sprintf("value-%d", i)
	}
	var pathFile = testutils.GenerateMeAString(20, 30)

	// setup the response
	for i := 0; i < 100; i++ {
		handler := func(num int) func(w http.ResponseWriter, r *http.Request) {
			return func(w http.ResponseWriter, r *http.Request) {
				w.Header().Add(headerKeyFunc(num), headerValueFunc(num))
				fmt.Fprintf(w, pathFile)
			}
		}(i)

		app.up.Handle("/path/"+strconv.Itoa(i), http.HandlerFunc(handler))
	}

	testFunc := func(t *testing.T, i, j int) {
		rec := httptest.NewRecorder()
		req, err := http.NewRequest("HEAD", "/path/"+strconv.Itoa(i), nil)
		if err != nil {
			t.Fatal(err)
		}
		app.cacheHandler.RequestHandle(context.Background(), rec, req)
		val := rec.Header().Get(headerKeyFunc(i))
		expVal := headerValueFunc(i)
		if val != expVal {
			t.Errorf("Expected header value %s and received %s", expVal, val)
		}
	}
	concurrentTestHelper(t, 20, 100, testFunc)
}
Exemple #7
0
func TestVeryFragmentedFile(t *testing.T) {
	t.Parallel()
	app := newTestApp(t)
	var file = "long"
	app.fsmap[file] = testutils.GenerateMeAString(1, 2000)
	defer app.cleanup()

	app.testRange(file, 5, 10)
	app.testRange(file, 5, 2)
	app.testRange(file, 2, 2)
	app.testRange(file, 20, 10)
	app.testRange(file, 30, 10)
	app.testRange(file, 40, 10)
	app.testRange(file, 50, 10)
	app.testRange(file, 60, 10)
	app.testRange(file, 70, 10)
	app.testRange(file, 50, 20)
	app.testRange(file, 200, 5)
	app.testFullRequest(file)
	app.testRange(file, 3, 1000)
}
		for pb.Next() {
			lru := aFullCache(b, benchCacheSize)
			if lru.stats().Objects() != benchCacheSize {
				b.Fatalf("expected %d objects in the full lru but got %d",
					benchCacheSize,
					lru.stats().Objects())
			}
		}
	})
}

var randPath = func() func() string {
	ch := make(chan string, 1000)
	go func() {
		for seed := int64(0); ; seed++ {
			ch <- testutils.GenerateMeAString(seed, 20)
		}
	}()
	return func() string {
		return <-ch
	}
}()

func BenchmarkLookupAndRemove(b *testing.B) {
	b.StopTimer()
	lru := aFullCache(b, benchCacheSize)
	var tooFast uint64
	b.StartTimer()
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			oi := getObjectIndexFor(randUint32(), "1.1", randPath())