Esempio n. 1
0
// BenchmarkFuzz is the benchmark version of TestFuzz. However, it will run
// several append and verify operations in parallel, if GOMAXPROC is set
// accordingly. Also, the storage options are set such that evictions,
// checkpoints, and purging will happen concurrently, too. This benchmark will
// have a very long runtime (up to minutes). You can use it as an actual
// benchmark. Run it like this:
//
// go test -cpu 1,2,4,8 -short -bench BenchmarkFuzz -benchmem
//
// You can also use it as a test for races. In that case, run it like this (will
// make things even slower):
//
// go test -race -cpu 8 -short -bench BenchmarkFuzz
func BenchmarkFuzz(b *testing.B) {
	b.StopTimer()
	rand.Seed(42)
	directory := test.NewTemporaryDirectory("test_storage", b)
	defer directory.Close()
	o := &MemorySeriesStorageOptions{
		MemoryChunks:               100,
		PersistenceRetentionPeriod: time.Hour,
		PersistenceStoragePath:     directory.Path(),
		CheckpointInterval:         3 * time.Second,
	}
	s, err := NewMemorySeriesStorage(o)
	if err != nil {
		b.Fatalf("Error creating storage: %s", err)
	}
	s.Start()
	defer s.Stop()
	b.StartTimer()

	b.RunParallel(func(pb *testing.PB) {
		var allSamples clientmodel.Samples
		for pb.Next() {
			newSamples := createRandomSamples()
			allSamples = append(allSamples, newSamples[:len(newSamples)/2]...)
			s.AppendSamples(newSamples[:len(newSamples)/2])
			verifyStorage(b, s, allSamples, o.PersistenceRetentionPeriod)
			allSamples = append(allSamples, newSamples[len(newSamples)/2:]...)
			s.AppendSamples(newSamples[len(newSamples)/2:])
			verifyStorage(b, s, allSamples, o.PersistenceRetentionPeriod)
		}
	})
}
Esempio n. 2
0
// This is copied from storage/metric/helpers_test.go, which is unfortunate but
// presently required to make things work.
func NewTestTieredStorage(t test.Tester) (storage *tiered.TieredStorage, closer test.Closer) {
	var directory test.TemporaryDirectory
	directory = test.NewTemporaryDirectory("test_tiered_storage", t)
	storage, err := tiered.NewTieredStorage(2500, 1000, 5*time.Second, 0*time.Second, directory.Path())

	if err != nil {
		if storage != nil {
			storage.Close()
		}
		directory.Close()
		t.Fatalf("Error creating storage: %s", err)
	}

	if storage == nil {
		directory.Close()
		t.Fatalf("storage == nil")
	}
	started := make(chan bool)
	go storage.Serve(started)
	<-started
	closer = &testTieredStorageCloser{
		storage:   storage,
		directory: directory,
	}
	return
}
Esempio n. 3
0
func (p preparer) Prepare(n string, f FixtureFactory) (t test.TemporaryDirectory) {
	t = test.NewTemporaryDirectory(n, p.tester)
	persistence, err := leveldb.NewLevelDBPersistence(leveldb.LevelDBOptions{
		Path:           t.Path(),
		CacheSizeBytes: cacheCapacity,
	})
	if err != nil {
		defer t.Close()
		p.tester.Fatal(err)
	}

	defer persistence.Close()

	for f.HasNext() {
		key, value := f.Next()

		switch v := value.(type) {
		case proto.Message:
			err = persistence.Put(key, v)
		case []byte:
			err = persistence.PutRaw(key, v)
		default:
			panic("illegal value type")
		}
		if err != nil {
			defer t.Close()
			p.tester.Fatal(err)
		}
	}

	return
}
Esempio n. 4
0
// TestLoop is just a smoke test for the loop method, if we can switch it on and
// off without disaster.
func TestLoop(t *testing.T) {
	samples := make(clientmodel.Samples, 1000)
	for i := range samples {
		samples[i] = &clientmodel.Sample{
			Timestamp: clientmodel.Timestamp(2 * i),
			Value:     clientmodel.SampleValue(float64(i) * 0.2),
		}
	}
	directory := test.NewTemporaryDirectory("test_storage", t)
	defer directory.Close()
	o := &MemorySeriesStorageOptions{
		MemoryChunks:               50,
		PersistenceRetentionPeriod: 24 * 7 * time.Hour,
		PersistenceStoragePath:     directory.Path(),
		CheckpointInterval:         250 * time.Millisecond,
	}
	storage, err := NewMemorySeriesStorage(o)
	if err != nil {
		t.Fatalf("Error creating storage: %s", err)
	}
	storage.Start()
	storage.AppendSamples(samples)
	time.Sleep(time.Second)
	storage.Stop()
}
Esempio n. 5
0
func TestLocking(t *testing.T) {
	dir := test.NewTemporaryDirectory("test_flock", t)
	defer dir.Close()

	fileName := filepath.Join(dir.Path(), "LOCK")

	if _, err := os.Stat(fileName); err == nil {
		t.Fatalf("File %q unexpectedly exists.", fileName)
	}

	lock, existed, err := New(fileName)
	if err != nil {
		t.Fatalf("Error locking file %q: %s", fileName, err)
	}
	if existed {
		t.Errorf("File %q reported as existing during locking.", fileName)
	}

	// File must now exist.
	if _, err := os.Stat(fileName); err != nil {
		t.Errorf("Could not stat file %q expected to exist: %s", fileName, err)
	}

	// Try to lock again.
	lockedAgain, existed, err := New(fileName)
	if err == nil {
		t.Fatalf("File %q locked twice.", fileName)
	}
	if lockedAgain != nil {
		t.Error("Unsuccessful locking did not return nil.")
	}
	if !existed {
		t.Errorf("Existing file %q not recognized.", fileName)
	}

	if err := lock.Release(); err != nil {
		t.Errorf("Error releasing lock for file %q: %s", fileName, err)
	}

	// File must still exist.
	if _, err := os.Stat(fileName); err != nil {
		t.Errorf("Could not stat file %q expected to exist: %s", fileName, err)
	}

	// Lock existing file.
	lock, existed, err = New(fileName)
	if err != nil {
		t.Fatalf("Error locking file %q: %s", fileName, err)
	}
	if !existed {
		t.Errorf("Existing file %q not recognized.", fileName)
	}

	if err := lock.Release(); err != nil {
		t.Errorf("Error releasing lock for file %q: %s", fileName, err)
	}
}
Esempio n. 6
0
func buildLevelDBTestPersistencesMaker(name string, t test.Tester) func() (MetricPersistence, test.Closer) {
	return func() (MetricPersistence, test.Closer) {
		temporaryDirectory := test.NewTemporaryDirectory("get_value_at_time", t)

		p, err := NewLevelDBMetricPersistence(temporaryDirectory.Path())
		if err != nil {
			t.Errorf("Could not start up LevelDB: %q\n", err)
		}

		return p, temporaryDirectory
	}
}
func testLevelDBStochastic(t test.Tester) {
	persistenceMaker := func() (MetricPersistence, test.Closer) {
		temporaryDirectory := test.NewTemporaryDirectory("test_leveldb_stochastic", t)

		p, err := NewLevelDBMetricPersistence(temporaryDirectory.Path())
		if err != nil {
			t.Errorf("Could not start up LevelDB: %q\n", err)
		}

		return p, temporaryDirectory
	}

	StochasticTests(persistenceMaker, t)
}
Esempio n. 8
0
func buildLevelDBTestPersistence(name string, f func(p MetricPersistence, t test.Tester)) func(t test.Tester) {
	return func(t test.Tester) {
		temporaryDirectory := test.NewTemporaryDirectory(fmt.Sprintf("test_leveldb_%s", name), t)
		defer temporaryDirectory.Close()

		p, err := NewLevelDBMetricPersistence(temporaryDirectory.Path())
		if err != nil {
			t.Errorf("Could not create LevelDB Metric Persistence: %q\n", err)
		}

		defer p.Close()

		f(p, t)
	}
}
Esempio n. 9
0
func newTestTieredStorage(t test.Tester) (storage Storage, closer test.Closer) {
	var directory test.TemporaryDirectory
	directory = test.NewTemporaryDirectory("test_tiered_storage", t)
	storage, err := NewTieredStorage(5000000, 2500, 1000, 5*time.Second, 15*time.Second, 0*time.Second, directory.Path())

	if err != nil {
		t.Fatalf("Error creating storage: %s", err)
	}

	if storage == nil {
		t.Fatalf("storage == nil")
	}

	go storage.Serve()
	closer = &testTieredStorageCloser{
		storage:   storage,
		directory: directory,
	}
	return
}
Esempio n. 10
0
func (p preparer) Prepare(n string, f FixtureFactory) (t test.TemporaryDirectory) {
	t = test.NewTemporaryDirectory(n, p.tester)
	persistence, err := leveldb.NewLevelDBPersistence(t.Path(), cacheCapacity, bitsPerBloomFilterEncoded)
	if err != nil {
		defer t.Close()
		p.tester.Fatal(err)
	}

	defer persistence.Close()

	for f.HasNext() {
		key, value := f.Next()

		err = persistence.Put(key, value)
		if err != nil {
			defer t.Close()
			p.tester.Fatal(err)
		}
	}

	return
}
Esempio n. 11
0
// NewTestStorage creates a storage instance backed by files in a temporary
// directory. The returned storage is already in serving state. Upon closing the
// returned test.Closer, the temporary directory is cleaned up.
func NewTestStorage(t test.T) (Storage, test.Closer) {
	directory := test.NewTemporaryDirectory("test_storage", t)
	o := &MemorySeriesStorageOptions{
		MemoryChunks:               1000000,
		PersistenceRetentionPeriod: 24 * 7 * time.Hour,
		PersistenceStoragePath:     directory.Path(),
		CheckpointInterval:         time.Hour,
	}
	storage, err := NewMemorySeriesStorage(o)
	if err != nil {
		directory.Close()
		t.Fatalf("Error creating storage: %s", err)
	}

	storage.Start()

	closer := &testStorageCloser{
		storage:   storage,
		directory: directory,
	}

	return storage, closer
}
Esempio n. 12
0
func TestExpressions(t *testing.T) {
	temporaryDirectory := test.NewTemporaryDirectory("rule_expression_tests", t)
	defer temporaryDirectory.Close()
	tieredStorage, err := metric.NewTieredStorage(5000, 5000, 100, time.Second*30, time.Second*1, time.Second*20, temporaryDirectory.Path())
	if err != nil {
		t.Fatalf("Error opening storage: %s", err)
	}
	go tieredStorage.Serve()

	ast.SetStorage(tieredStorage)

	storeMatrix(tieredStorage, testMatrix)
	tieredStorage.Flush()

	for i, exprTest := range expressionTests {
		expectedLines := annotateWithTime(exprTest.output)

		testExpr, err := LoadExprFromString(exprTest.expr)

		if err != nil {
			if exprTest.shouldFail {
				continue
			}
			t.Errorf("%d Error during parsing: %v", i, err)
			t.Errorf("%d Expression: %v", i, exprTest.expr)
		} else {
			if exprTest.shouldFail {
				t.Errorf("%d Test should fail, but didn't", i)
			}
			failed := false
			resultStr := ast.EvalToString(testExpr, testEvalTime, ast.TEXT)
			resultLines := strings.Split(resultStr, "\n")

			if len(exprTest.output) != len(resultLines) {
				t.Errorf("%d Number of samples in expected and actual output don't match", i)
				failed = true
			}

			if exprTest.checkOrder {
				for j, expectedSample := range expectedLines {
					if resultLines[j] != expectedSample {
						t.Errorf("%d.%d Expected sample '%v', got '%v'", i, j, resultLines[j], expectedSample)
						failed = true
					}
				}
			} else {
				for j, expectedSample := range expectedLines {
					found := false
					for _, actualSample := range resultLines {
						if actualSample == expectedSample {
							found = true
						}
					}
					if !found {
						t.Errorf("%d.%d Couldn't find expected sample in output: '%v'", i, j, expectedSample)
						failed = true
					}
				}
			}

			analyzer := ast.NewQueryAnalyzer()
			analyzer.AnalyzeQueries(testExpr)
			if exprTest.fullRanges != len(analyzer.FullRanges) {
				t.Errorf("%d Count of full ranges didn't match: %v vs %v", i, exprTest.fullRanges, len(analyzer.FullRanges))
				failed = true
			}
			if exprTest.intervalRanges != len(analyzer.IntervalRanges) {
				t.Errorf("%d Count of interval ranges didn't match: %v vs %v", i, exprTest.intervalRanges, len(analyzer.IntervalRanges))
				failed = true
			}

			if failed {
				t.Errorf("%d Expression: %v\n%v", i, exprTest.expr, vectorComparisonString(expectedLines, resultLines))
			}
		}
	}
}