// benchmarkWritePoints benchmarks writing new series to a shard.
// mCnt - measurement count
// tkCnt - tag key count
// tvCnt - tag value count (values per tag)
// pntCnt - points per series. # of series = mCnt * (tvCnt ^ tkCnt)
func benchmarkWritePoints(b *testing.B, mCnt, tkCnt, tvCnt, pntCnt int) {
// Generate test series (measurements + unique tag sets).
series := genTestSeries(mCnt, tkCnt, tvCnt)
// Create index for the shard to use.
index := tsdb.NewDatabaseIndex()
// Generate point data to write to the shard.
points := []tsdb.Point{}
for _, s := range series {
for val := 0.0; val < float64(pntCnt); val++ {
p := tsdb.NewPoint(s.Measurement, s.Series.Tags, map[string]interface{}{"value": val}, time.Now())
points = append(points, p)
}
}
// Stop & reset timers and mem-stats before the main benchmark loop.
b.StopTimer()
b.ResetTimer()
// Run the benchmark loop.
for n := 0; n < b.N; n++ {
tmpDir, _ := ioutil.TempDir("", "shard_test")
tmpShard := path.Join(tmpDir, "shard")
shard := tsdb.NewShard(1, index, tmpShard, tsdb.NewEngineOptions())
shard.Open()
b.StartTimer()
// Call the function being benchmarked.
chunkedWrite(shard, points)
b.StopTimer()
shard.Close()
os.RemoveAll(tmpDir)
}
}
func TestShardWriteAddNewField(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex()
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error openeing shard: %s", err.Error())
}
defer sh.Close()
pt := tsdb.NewPoint(
"cpu",
map[string]string{"host": "server"},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]tsdb.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
pt = tsdb.NewPoint(
"cpu",
map[string]string{"host": "server"},
map[string]interface{}{"value": 1.0, "value2": 2.0},
time.Unix(1, 2),
)
err = sh.WritePoints([]tsdb.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
if !reflect.DeepEqual(index.Names(), []string{"cpu"}) {
t.Fatalf("measurement names in shard didn't match")
}
if index.SeriesN() != 1 {
t.Fatalf("series wasn't in index")
}
seriesTags := index.Series(string(pt.Key())).Tags
if len(seriesTags) != len(pt.Tags()) || pt.Tags()["host"] != seriesTags["host"] {
t.Fatalf("tags weren't properly saved to series index: %v, %v", pt.Tags(), seriesTags)
}
if !reflect.DeepEqual(index.Measurement("cpu").TagKeys(), []string{"host"}) {
t.Fatalf("tag key wasn't saved to measurement index")
}
if len(index.Measurement("cpu").FieldNames()) != 2 {
t.Fatalf("field names wasn't saved to measurement index")
}
}
// Ensure that deletes only sent to the WAL will clear out the data from the cache on restart
func TestDevEngine_DeleteWALLoadMetadata(t *testing.T) {
// Generate temporary file.
f, _ := ioutil.TempFile("", "tsm")
f.Close()
os.Remove(f.Name())
walPath := filepath.Join(f.Name(), "wal")
os.MkdirAll(walPath, 0777)
defer os.RemoveAll(f.Name())
// Create a few points.
p1 := parsePoint("cpu,host=A value=1.1 1000000000")
p2 := parsePoint("cpu,host=B value=1.2 2000000000")
// Write those points to the engine.
e := NewDevEngine(f.Name(), walPath, tsdb.NewEngineOptions()).(*DevEngine)
if err := e.Open(); err != nil {
t.Fatalf("failed to open tsm1 engine: %s", err.Error())
}
if err := e.WritePoints([]models.Point{p1, p2}, nil, nil); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
if err := e.DeleteSeries([]string{"cpu,host=A"}); err != nil {
t.Fatalf("failed to delete series: %s", err.Error())
}
// ensure we can close and load index from the WAL
if err := e.Close(); err != nil {
t.Fatalf("error closing: %s", err.Error())
}
e = NewDevEngine(f.Name(), walPath, tsdb.NewEngineOptions()).(*DevEngine)
if err := e.Open(); err != nil {
t.Fatalf("failed to open tsm1 engine: %s", err.Error())
}
fmt.Println(e.Cache.store)
if exp, got := 0, len(e.Cache.Values(SeriesFieldKey("cpu,host=A", "value"))); exp != got {
t.Fatalf("unexpected number of values: got: %d. exp: %d", got, exp)
}
if exp, got := 1, len(e.Cache.Values(SeriesFieldKey("cpu,host=B", "value"))); exp != got {
t.Fatalf("unexpected number of values: got: %d. exp: %d", got, exp)
}
}
func mustCreateShard(dir string) *tsdb.Shard {
tmpShard := path.Join(dir, "shard")
index := tsdb.NewDatabaseIndex()
sh := tsdb.NewShard(index, tmpShard, tsdb.NewEngineOptions())
if err := sh.Open(); err != nil {
panic(fmt.Sprintf("error opening shard: %s", err.Error()))
}
return sh
}
// Ensure an engine containing cached values responds correctly to queries.
func TestDevEngine_QueryCache_Ascending(t *testing.T) {
// Generate temporary file.
f, _ := ioutil.TempFile("", "tsm")
f.Close()
os.Remove(f.Name())
walPath := filepath.Join(f.Name(), "wal")
os.MkdirAll(walPath, 0777)
defer os.RemoveAll(f.Name())
// Create a few points.
p1 := parsePoint("cpu,host=A value=1.1 1000000000")
p2 := parsePoint("cpu,host=A value=1.2 2000000000")
p3 := parsePoint("cpu,host=A value=1.3 3000000000")
// Write those points to the engine.
e := NewDevEngine(f.Name(), walPath, tsdb.NewEngineOptions())
if err := e.Open(); err != nil {
t.Fatalf("failed to open tsm1 engine: %s", err.Error())
}
if err := e.WritePoints([]models.Point{p1, p2, p3}, nil, nil); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
// Start a query transactions and get a cursor.
tx := devTx{engine: e.(*DevEngine)}
ascCursor := tx.Cursor("cpu,host=A", []string{"value"}, nil, true)
k, v := ascCursor.SeekTo(1)
if k != 1000000000 {
t.Fatalf("failed to seek to before first key: %v %v", k, v)
}
k, v = ascCursor.SeekTo(1000000000)
if k != 1000000000 {
t.Fatalf("failed to seek to first key: %v %v", k, v)
}
k, v = ascCursor.Next()
if k != 2000000000 {
t.Fatalf("failed to get next key: %v %v", k, v)
}
k, v = ascCursor.Next()
if k != 3000000000 {
t.Fatalf("failed to get next key: %v %v", k, v)
}
k, v = ascCursor.Next()
if k != -1 {
t.Fatalf("failed to get next key: %v %v", k, v)
}
k, v = ascCursor.SeekTo(4000000000)
if k != -1 {
t.Fatalf("failed to seek to past last key: %v %v", k, v)
}
}
func mustCreateShard(dir string) *tsdb.Shard {
tmpShard := path.Join(dir, "shard")
tmpWal := path.Join(dir, "wal")
index := tsdb.NewDatabaseIndex()
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(dir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
panic(fmt.Sprintf("error opening shard: %s", err.Error()))
}
return sh
}
// MustOpenShard returns a temporary, opened shard.
func MustOpenShard(id uint64) *Shard {
path, err := ioutil.TempDir("", "copier-")
if err != nil {
panic(err)
}
sh := &Shard{
Shard: tsdb.NewShard(id,
tsdb.NewDatabaseIndex(),
filepath.Join(path, "data"),
filepath.Join(path, "wal"),
tsdb.NewEngineOptions(),
),
path: path,
}
if err := sh.Open(); err != nil {
sh.Close()
panic(err)
}
return sh
}
// OpenDefaultEngine returns an open Engine with default options.
func OpenDefaultEngine() *Engine { return OpenEngine(tsdb.NewEngineOptions()) }
func TestShardWriteAndIndex(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
index := tsdb.NewDatabaseIndex()
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error openeing shard: %s", err.Error())
}
pt := tsdb.NewPoint(
"cpu",
map[string]string{"host": "server"},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]tsdb.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
pt.SetTime(time.Unix(2, 3))
err = sh.WritePoints([]tsdb.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
validateIndex := func() {
if !reflect.DeepEqual(index.Names(), []string{"cpu"}) {
t.Fatalf("measurement names in shard didn't match")
}
if index.SeriesN() != 1 {
t.Fatalf("series wasn't in index")
}
seriesTags := index.Series(string(pt.Key())).Tags
if len(seriesTags) != len(pt.Tags()) || pt.Tags()["host"] != seriesTags["host"] {
t.Fatalf("tags weren't properly saved to series index: %v, %v", pt.Tags(), seriesTags)
}
if !reflect.DeepEqual(index.Measurement("cpu").TagKeys(), []string{"host"}) {
t.Fatalf("tag key wasn't saved to measurement index")
}
}
validateIndex()
// ensure the index gets loaded after closing and opening the shard
sh.Close()
index = tsdb.NewDatabaseIndex()
sh = tsdb.NewShard(1, index, tmpShard, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error openeing shard: %s", err.Error())
}
validateIndex()
// and ensure that we can still write data
pt.SetTime(time.Unix(2, 6))
err = sh.WritePoints([]tsdb.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
}
// Ensure that the engine will backup any TSM files created since the passed in time
func TestDevEngine_Backup(t *testing.T) {
// Generate temporary file.
f, _ := ioutil.TempFile("", "tsm")
f.Close()
os.Remove(f.Name())
walPath := filepath.Join(f.Name(), "wal")
os.MkdirAll(walPath, 0777)
defer os.RemoveAll(f.Name())
// Create a few points.
p1 := parsePoint("cpu,host=A value=1.1 1000000000")
p2 := parsePoint("cpu,host=B value=1.2 2000000000")
p3 := parsePoint("cpu,host=C value=1.3 3000000000")
// Write those points to the engine.
e := NewDevEngine(f.Name(), walPath, tsdb.NewEngineOptions()).(*DevEngine)
// mock the planner so compactions don't run during the test
e.CompactionPlan = &mockPlanner{}
if err := e.Open(); err != nil {
t.Fatalf("failed to open tsm1 engine: %s", err.Error())
}
if err := e.WritePoints([]models.Point{p1}, nil, nil); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
if err := e.WriteSnapshot(); err != nil {
t.Fatalf("failed to snapshot: %s", err.Error())
}
if err := e.WritePoints([]models.Point{p2}, nil, nil); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
b := bytes.NewBuffer(nil)
if err := e.Backup(b, "", time.Unix(0, 0)); err != nil {
t.Fatalf("failed to backup: %s", err.Error())
}
tr := tar.NewReader(b)
if len(e.FileStore.files) != 2 {
t.Fatalf("file count wrong: exp: %d, got: %d", 2, len(e.FileStore.files))
}
for _, f := range e.FileStore.files {
th, err := tr.Next()
if err != nil {
t.Fatalf("failed reading header: %s", err.Error())
}
if !strings.Contains(f.Path(), th.Name) || th.Name == "" {
t.Fatalf("file name doesn't match:\n\tgot: %s\n\texp: %s", th.Name, f.Path())
}
}
lastBackup := time.Now()
// we have to sleep for a second because last modified times only have second level precision.
// so this test won't work properly unless the file is at least a second past the last one
time.Sleep(time.Second)
if err := e.WritePoints([]models.Point{p3}, nil, nil); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
b = bytes.NewBuffer(nil)
if err := e.Backup(b, "", lastBackup); err != nil {
t.Fatalf("failed to backup: %s", err.Error())
}
tr = tar.NewReader(b)
th, err := tr.Next()
if err != nil {
t.Fatalf("error getting next tar header: %s", err.Error())
}
mostRecentFile := e.FileStore.files[len(e.FileStore.files)-1].Path()
if !strings.Contains(mostRecentFile, th.Name) || th.Name == "" {
t.Fatalf("file name doesn't match:\n\tgot: %s\n\texp: %s", th.Name, mostRecentFile)
}
}
func TestDevEngine_LoadMetadataIndex(t *testing.T) {
// Generate temporary file.
f, _ := ioutil.TempFile("", "tsm")
f.Close()
os.Remove(f.Name())
walPath := filepath.Join(f.Name(), "wal")
os.MkdirAll(walPath, 0777)
defer os.RemoveAll(f.Name())
// Create a few points.
p1 := parsePoint("cpu,host=A value=1.1 1000000000")
p2 := parsePoint("cpu,host=B value=1.2 2000000000")
// Write those points to the engine.
e := NewDevEngine(f.Name(), walPath, tsdb.NewEngineOptions()).(*DevEngine)
if err := e.Open(); err != nil {
t.Fatalf("failed to open tsm1 engine: %s", err.Error())
}
if err := e.WritePoints([]models.Point{p1}, nil, nil); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
// ensure we can close and load index from the WAL
if err := e.Close(); err != nil {
t.Fatalf("error closing: %s", err.Error())
}
if err := e.Open(); err != nil {
t.Fatalf("error opening: %s", err.Error())
}
// Load metadata index.
index := tsdb.NewDatabaseIndex()
if err := e.LoadMetadataIndex(nil, index, make(map[string]*tsdb.MeasurementFields)); err != nil {
t.Fatal(err)
}
// Verify index is correct.
if m := index.Measurement("cpu"); m == nil {
t.Fatal("measurement not found")
} else if s := m.SeriesByID(1); s.Key != "cpu,host=A" || !reflect.DeepEqual(s.Tags, map[string]string{"host": "A"}) {
t.Fatalf("unexpected series: %q / %#v", s.Key, s.Tags)
}
// write the snapshot, ensure we can close and load index from TSM
if err := e.WriteSnapshot(); err != nil {
t.Fatalf("error writing snapshot: %s", err.Error())
}
// ensure we can close and load index from the WAL
if err := e.Close(); err != nil {
t.Fatalf("error closing: %s", err.Error())
}
if err := e.Open(); err != nil {
t.Fatalf("error opening: %s", err.Error())
}
// Load metadata index.
index = tsdb.NewDatabaseIndex()
if err := e.LoadMetadataIndex(nil, index, make(map[string]*tsdb.MeasurementFields)); err != nil {
t.Fatal(err)
}
// Verify index is correct.
if m := index.Measurement("cpu"); m == nil {
t.Fatal("measurement not found")
} else if s := m.SeriesByID(1); s.Key != "cpu,host=A" || !reflect.DeepEqual(s.Tags, map[string]string{"host": "A"}) {
t.Fatalf("unexpected series: %q / %#v", s.Key, s.Tags)
}
// write a new point and ensure we can close and load index from TSM and WAL
if err := e.WritePoints([]models.Point{p2}, nil, nil); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
// ensure we can close and load index from the TSM and WAL
if err := e.Close(); err != nil {
t.Fatalf("error closing: %s", err.Error())
}
if err := e.Open(); err != nil {
t.Fatalf("error opening: %s", err.Error())
}
// Load metadata index.
index = tsdb.NewDatabaseIndex()
if err := e.LoadMetadataIndex(nil, index, make(map[string]*tsdb.MeasurementFields)); err != nil {
t.Fatal(err)
}
// Verify index is correct.
if m := index.Measurement("cpu"); m == nil {
t.Fatal("measurement not found")
} else if s := m.SeriesByID(1); s.Key != "cpu,host=A" || !reflect.DeepEqual(s.Tags, map[string]string{"host": "A"}) {
t.Fatalf("unexpected series: %q / %#v", s.Key, s.Tags)
} else if s := m.SeriesByID(2); s.Key != "cpu,host=B" || !reflect.DeepEqual(s.Tags, map[string]string{"host": "B"}) {
t.Fatalf("unexpected series: %q / %#v", s.Key, s.Tags)
}
}