func TestMakeKeyEscaped(t *testing.T) {
if exp, got := `cpu\ load`, models.MakeKey([]byte(`cpu\ load`), models.Tags{}); string(got) != exp {
t.Errorf("MakeKey() mismatch.\ngot %v\nexp %v", got, exp)
}
if exp, got := `cpu\ load`, models.MakeKey([]byte(`cpu load`), models.Tags{}); string(got) != exp {
t.Errorf("MakeKey() mismatch.\ngot %v\nexp %v", got, exp)
}
if exp, got := `cpu\,load`, models.MakeKey([]byte(`cpu\,load`), models.Tags{}); string(got) != exp {
t.Errorf("MakeKey() mismatch.\ngot %v\nexp %v", got, exp)
}
if exp, got := `cpu\,load`, models.MakeKey([]byte(`cpu,load`), models.Tags{}); string(got) != exp {
t.Errorf("MakeKey() mismatch.\ngot %v\nexp %v", got, exp)
}
}
// Ensure the engine can write series metadata and reload it.
func TestEngine_LoadMetadataIndex_Series(t *testing.T) {
e := OpenDefaultEngine()
defer e.Close()
// Setup mock that writes the index
seriesToCreate := []*tsdb.SeriesCreate{
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "server0"})), map[string]string{"host": "server0"})},
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "server1"})), map[string]string{"host": "server1"})},
{Series: tsdb.NewSeries("series with spaces", nil)},
}
e.PointsWriter.WritePointsFn = func(a []models.Point) error { return e.WriteIndex(nil, nil, seriesToCreate) }
// Write series metadata.
if err := e.WritePoints(nil, nil, seriesToCreate); err != nil {
t.Fatal(err)
}
// Load metadata index.
index := tsdb.NewDatabaseIndex()
if err := e.LoadMetadataIndex(index, make(map[string]*tsdb.MeasurementFields)); err != nil {
t.Fatal(err)
}
// Verify index is correct.
if m := index.Measurement("cpu"); m == nil {
t.Fatal("measurement not found")
} else if s := m.SeriesByID(1); s.Key != "cpu,host=server0" || !reflect.DeepEqual(s.Tags, map[string]string{"host": "server0"}) {
t.Fatalf("unexpected series: %q / %#v", s.Key, s.Tags)
} else if s = m.SeriesByID(2); s.Key != "cpu,host=server1" || !reflect.DeepEqual(s.Tags, map[string]string{"host": "server1"}) {
t.Fatalf("unexpected series: %q / %#v", s.Key, s.Tags)
}
if m := index.Measurement("series with spaces"); m == nil {
t.Fatal("measurement not found")
} else if s := m.SeriesByID(3); s.Key != "series with spaces" {
t.Fatalf("unexpected series: %q", s.Key)
}
}
func TestEngine_Deletes(t *testing.T) {
e := OpenDefaultEngine()
defer e.Close()
fields := []string{"value"}
// Create metadata.
mf := &tsdb.MeasurementFields{Fields: make(map[string]*tsdb.Field)}
mf.CreateFieldIfNotExists("value", influxql.Float, false)
atag := map[string]string{"host": "A"}
btag := map[string]string{"host": "B"}
seriesToCreate := []*tsdb.SeriesCreate{
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), atag)), atag)},
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), btag)), btag)},
}
p1 := parsePoint("cpu,host=A value=1.1 1000000001")
p2 := parsePoint("cpu,host=A value=1.2 2000000001")
p3 := parsePoint("cpu,host=B value=2.1 1000000000")
p4 := parsePoint("cpu,host=B value=2.1 2000000000")
e.SkipCompaction = true
e.WAL.SkipCache = false
if err := e.WritePoints([]models.Point{p1, p3}, map[string]*tsdb.MeasurementFields{"cpu": mf}, seriesToCreate); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
func() {
tx, _ := e.Begin(false)
defer tx.Rollback()
c := tx.Cursor("cpu,host=A", fields, nil, true)
k, _ := c.SeekTo(0)
if k != p1.UnixNano() {
t.Fatalf("time wrong:\n\texp:%d\n\tgot:%d\n", p1.UnixNano(), k)
}
}()
if err := e.DeleteSeries([]string{"cpu,host=A"}); err != nil {
t.Fatalf("failed to delete series: %s", err.Error())
}
func() {
tx, _ := e.Begin(false)
defer tx.Rollback()
c := tx.Cursor("cpu,host=B", fields, nil, true)
k, _ := c.SeekTo(0)
if k != p3.UnixNano() {
t.Fatalf("time wrong:\n\texp:%d\n\tgot:%d\n", p1.UnixNano(), k)
}
c = tx.Cursor("cpu,host=A", fields, nil, true)
k, _ = c.SeekTo(0)
if k != tsdb.EOF {
t.Fatal("expected EOF", k)
}
}()
if err := e.WritePoints([]models.Point{p2, p4}, nil, nil); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
if err := e.WAL.Flush(); err != nil {
t.Fatalf("error flushing wal: %s", err.Error())
}
func() {
tx, _ := e.Begin(false)
defer tx.Rollback()
c := tx.Cursor("cpu,host=A", fields, nil, true)
k, _ := c.SeekTo(0)
if k != p2.UnixNano() {
t.Fatalf("time wrong:\n\texp:%d\n\tgot:%d\n", p1.UnixNano(), k)
}
}()
if err := e.DeleteSeries([]string{"cpu,host=A"}); err != nil {
t.Fatalf("failed to delete series: %s", err.Error())
}
// we already know the delete on the wal works. open and close so
// the wal flushes to the index. To verify that the delete gets
// persisted and will go all the way through the index
if err := e.Engine.Close(); err != nil {
t.Fatalf("error closing: %s", err.Error())
}
if err := e.Open(); err != nil {
t.Fatalf("error opening: %s", err.Error())
}
verify := func() {
tx, _ := e.Begin(false)
defer tx.Rollback()
c := tx.Cursor("cpu,host=B", fields, nil, true)
k, _ := c.SeekTo(0)
if k != p3.UnixNano() {
t.Fatalf("time wrong:\n\texp:%d\n\tgot:%d\n", p1.UnixNano(), k)
}
c = tx.Cursor("cpu,host=A", fields, nil, true)
k, _ = c.SeekTo(0)
if k != tsdb.EOF {
t.Fatal("expected EOF")
}
}
fmt.Println("verify 1")
verify()
// open and close to verify thd delete was persisted
if err := e.Engine.Close(); err != nil {
t.Fatalf("error closing: %s", err.Error())
}
if err := e.Open(); err != nil {
t.Fatalf("error opening: %s", err.Error())
}
fmt.Println("verify 2")
verify()
if err := e.DeleteSeries([]string{"cpu,host=B"}); err != nil {
t.Fatalf("failed to delete series: %s", err.Error())
}
func() {
tx, _ := e.Begin(false)
defer tx.Rollback()
c := tx.Cursor("cpu,host=B", fields, nil, true)
k, _ := c.SeekTo(0)
if k != tsdb.EOF {
t.Fatal("expected EOF")
}
}()
if err := e.WAL.Flush(); err != nil {
t.Fatalf("error flushing: %s", err.Error())
}
func() {
tx, _ := e.Begin(false)
defer tx.Rollback()
c := tx.Cursor("cpu,host=B", fields, nil, true)
k, _ := c.SeekTo(0)
if k != tsdb.EOF {
t.Fatal("expected EOF")
}
}()
// open and close to verify thd delete was persisted
if err := e.Engine.Close(); err != nil {
t.Fatalf("error closing: %s", err.Error())
}
if err := e.Open(); err != nil {
t.Fatalf("error opening: %s", err.Error())
}
func() {
tx, _ := e.Begin(false)
defer tx.Rollback()
c := tx.Cursor("cpu,host=B", fields, nil, true)
k, _ := c.SeekTo(0)
if k != tsdb.EOF {
t.Fatal("expected EOF")
}
}()
}
// Ensure points can be written to the engine and queried in reverse order.
func TestEngine_WritePoints_Reverse(t *testing.T) {
e := OpenDefaultEngine()
defer e.Close()
// Create metadata.
mf := &tsdb.MeasurementFields{Fields: make(map[string]*tsdb.Field)}
mf.CreateFieldIfNotExists("value", influxql.Float)
seriesToCreate := []*tsdb.SeriesCreate{
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("temperature"), nil)), nil)},
}
// Parse point.
points, err := models.ParsePointsWithPrecision([]byte("temperature value=100 0"), time.Now().UTC(), "s")
if err != nil {
t.Fatal(err)
} else if data, err := mf.Codec.EncodeFields(points[0].Fields()); err != nil {
t.Fatal(err)
} else {
points[0].SetData(data)
}
// Write original value.
if err := e.WritePoints(points, map[string]*tsdb.MeasurementFields{"temperature": mf}, seriesToCreate); err != nil {
t.Fatal(err)
}
// Flush to disk.
if err := e.Flush(0); err != nil {
t.Fatal(err)
}
// Parse new point.
points, err = models.ParsePointsWithPrecision([]byte("temperature value=200 1"), time.Now().UTC(), "s")
if err != nil {
t.Fatal(err)
} else if data, err := mf.Codec.EncodeFields(points[0].Fields()); err != nil {
t.Fatal(err)
} else {
points[0].SetData(data)
}
// Write the new points existing value.
if err := e.WritePoints(points, nil, nil); err != nil {
t.Fatal(err)
}
// Ensure only the updated value is read.
tx := e.MustBegin(false)
defer tx.Rollback()
c := tx.Cursor("temperature", []string{"value"}, mf.Codec, false)
if k, _ := c.SeekTo(math.MaxInt64); k != time.Unix(1, 0).UnixNano() {
t.Fatalf("unexpected key: %v", k)
} else if k, v := c.Next(); k != time.Unix(0, 0).UnixNano() {
t.Fatalf("unexpected key: %v", k)
} else if v == nil || v.(float64) != 100 {
t.Errorf("unexpected value: %#v", v)
}
if k, v := c.Next(); k != tsdb.EOF {
t.Fatalf("unexpected key/value: %#v / %#v", k, v)
}
}
func TestWAL_DeleteSeries(t *testing.T) {
log := openTestWAL()
defer log.Close()
defer os.RemoveAll(log.path)
codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{
"value": {
ID: uint8(1),
Name: "value",
Type: influxql.Float,
},
})
var seriesToIndex []*tsdb.SeriesCreate
var points map[string][][]byte
log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error {
points = pointsByKey
seriesToIndex = append(seriesToIndex, seriesToCreate...)
return nil
}}
seriesToCreate := []*tsdb.SeriesCreate{
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "A"})), map[string]string{"host": "A"})},
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "B"})), map[string]string{"host": "B"})},
}
if err := log.Open(); err != nil {
t.Fatalf("couldn't open wal: %s", err.Error())
}
// test that we can write to two different series
p1 := parsePoint("cpu,host=A value=23.2 1", codec)
p2 := parsePoint("cpu,host=B value=0.9 2", codec)
p3 := parsePoint("cpu,host=A value=25.3 4", codec)
p4 := parsePoint("cpu,host=B value=1.0 3", codec)
if err := log.WritePoints([]models.Point{p1, p2, p3, p4}, nil, seriesToCreate); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
// ensure data is there
c := log.Cursor("cpu,host=A", []string{"value"}, codec, true)
if k, _ := c.Next(); k != 1 {
t.Fatal("expected data point for cpu,host=A")
}
c = log.Cursor("cpu,host=B", []string{"value"}, codec, true)
if k, _ := c.Next(); k != 2 {
t.Fatal("expected data point for cpu,host=B")
}
// delete the series and ensure metadata was flushed and data is gone
if err := log.DeleteSeries([]string{"cpu,host=B"}); err != nil {
t.Fatalf("error deleting series: %s", err.Error())
}
// ensure data is there
if len(points["cpu,host=A"]) != 2 {
t.Fatal("expected cpu,host=A to be flushed to the index")
}
if len(points["cpu,host=B"]) != 0 {
t.Fatal("expected cpu,host=B to have no points in index")
}
c = log.Cursor("cpu,host=A", []string{"value"}, codec, true)
if k, _ := c.Next(); k != tsdb.EOF {
t.Fatal("expected data to be out of the cache cpu,host=A")
}
// ensure series is deleted
c = log.Cursor("cpu,host=B", []string{"value"}, codec, true)
if k, _ := c.Next(); k != tsdb.EOF {
t.Fatal("expected no data for cpu,host=B")
}
// ensure that they were actually flushed to the index. do it this way because the annoying deepequal doessn't really work for these
for i, s := range seriesToCreate {
if seriesToIndex[i].Measurement != s.Measurement {
t.Fatal("expected measurement to be the same")
}
if seriesToIndex[i].Series.Key != s.Series.Key {
t.Fatal("expected series key to be the same")
}
if !reflect.DeepEqual(seriesToIndex[i].Series.Tags, s.Series.Tags) {
t.Fatal("expected series tags to be the same")
}
}
// close and re-open the WAL to ensure that the data didn't show back up
if err := log.Close(); err != nil {
t.Fatalf("error closing log: %s", err.Error())
}
points = make(map[string][][]byte)
if err := log.Open(); err != nil {
t.Fatalf("error opening log: %s", err.Error())
}
// ensure data wasn't flushed on open
if len(points) != 0 {
t.Fatal("expected no data to be flushed on open")
}
}
func TestWAL_SeriesAndFieldsGetPersisted(t *testing.T) {
log := openTestWAL()
defer log.Close()
defer os.RemoveAll(log.path)
if err := log.Open(); err != nil {
t.Fatalf("couldn't open wal: %s", err.Error())
}
codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{
"value": {
ID: uint8(1),
Name: "value",
Type: influxql.Float,
},
})
var measurementsToIndex map[string]*tsdb.MeasurementFields
var seriesToIndex []*tsdb.SeriesCreate
log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error {
measurementsToIndex = measurementFieldsToSave
seriesToIndex = append(seriesToIndex, seriesToCreate...)
return nil
}}
// test that we can write to two different series
p1 := parsePoint("cpu,host=A value=23.2 1", codec)
p2 := parsePoint("cpu,host=A value=25.3 4", codec)
p3 := parsePoint("cpu,host=B value=1.0 1", codec)
seriesToCreate := []*tsdb.SeriesCreate{
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "A"})), map[string]string{"host": "A"})},
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "B"})), map[string]string{"host": "B"})},
}
measaurementsToCreate := map[string]*tsdb.MeasurementFields{
"cpu": {
Fields: map[string]*tsdb.Field{
"value": {ID: 1, Name: "value"},
},
},
}
if err := log.WritePoints([]models.Point{p1, p2, p3}, measaurementsToCreate, seriesToCreate); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
// now close it and see if loading the metadata index will populate the measurement and series info
log.Close()
idx := tsdb.NewDatabaseIndex()
mf := make(map[string]*tsdb.MeasurementFields)
if err := log.LoadMetadataIndex(idx, mf); err != nil {
t.Fatalf("error loading metadata index: %s", err.Error())
}
s := idx.Series("cpu,host=A")
if s == nil {
t.Fatal("expected to find series cpu,host=A in index")
}
s = idx.Series("cpu,host=B")
if s == nil {
t.Fatal("expected to find series cpu,host=B in index")
}
m := mf["cpu"]
if m == nil {
t.Fatal("expected to find measurement fields for cpu", mf)
}
if m.Fields["value"] == nil {
t.Fatal("expected to find field definition for 'value'")
}
// ensure that they were actually flushed to the index. do it this way because the annoying deepequal doessn't really work for these
for i, s := range seriesToCreate {
if seriesToIndex[i].Measurement != s.Measurement {
t.Fatal("expected measurement to be the same")
}
if seriesToIndex[i].Series.Key != s.Series.Key {
t.Fatal("expected series key to be the same")
}
if !reflect.DeepEqual(seriesToIndex[i].Series.Tags, s.Series.Tags) {
t.Fatal("expected series tags to be the same")
}
}
// ensure that the measurement fields were flushed to the index
for k, v := range measaurementsToCreate {
m := measurementsToIndex[k]
if m == nil {
t.Fatalf("measurement %s wasn't indexed", k)
}
if !reflect.DeepEqual(m.Fields, v.Fields) {
t.Fatal("measurement fields not equal")
}
}
// now open and close the log and try to reload the metadata index, which should now be empty
if err := log.Open(); err != nil {
t.Fatalf("error opening log: %s", err.Error())
}
if err := log.Close(); err != nil {
t.Fatalf("error closing log: %s", err.Error())
}
idx = tsdb.NewDatabaseIndex()
mf = make(map[string]*tsdb.MeasurementFields)
if err := log.LoadMetadataIndex(idx, mf); err != nil {
t.Fatalf("error loading metadata index: %s", err.Error())
}
if len(idx.Measurements()) != 0 || len(mf) != 0 {
t.Fatal("expected index and measurement fields to be empty")
}
}
// Ensure points can be written to the engine and queried.
func TestEngine_WritePoints(t *testing.T) {
e := OpenDefaultEngine()
defer e.Close()
// Create metadata.
mf := &tsdb.MeasurementFields{Fields: make(map[string]*tsdb.Field)}
mf.CreateFieldIfNotExists("value", influxql.Float)
seriesToCreate := []*tsdb.SeriesCreate{
{Series: tsdb.NewSeries(string(models.MakeKey([]byte("temperature"), nil)), nil)},
}
// Parse point.
points, err := models.ParsePointsWithPrecision([]byte("temperature value=100 1434059627"), time.Now().UTC(), "s")
if err != nil {
t.Fatal(err)
} else if data, err := mf.Codec.EncodeFields(points[0].Fields()); err != nil {
t.Fatal(err)
} else {
points[0].SetData(data)
}
// Write original value.
if err := e.WritePoints(points, map[string]*tsdb.MeasurementFields{"temperature": mf}, seriesToCreate); err != nil {
t.Fatal(err)
}
// Flush to disk.
if err := e.Flush(0); err != nil {
t.Fatal(err)
}
// Parse new point.
points, err = models.ParsePointsWithPrecision([]byte("temperature value=200 1434059627"), time.Now().UTC(), "s")
if err != nil {
t.Fatal(err)
} else if data, err := mf.Codec.EncodeFields(points[0].Fields()); err != nil {
t.Fatal(err)
} else {
points[0].SetData(data)
}
// Update existing value.
if err := e.WritePoints(points, nil, nil); err != nil {
t.Fatal(err)
}
// Ensure only the updated value is read.
tx := e.MustBegin(false)
defer tx.Rollback()
c := tx.Cursor("temperature", tsdb.Forward)
if k, v := c.Seek([]byte{0}); !bytes.Equal(k, u64tob(uint64(time.Unix(1434059627, 0).UnixNano()))) {
t.Fatalf("unexpected key: %#v", k)
} else if m, err := mf.Codec.DecodeFieldsWithNames(v); err != nil {
t.Fatal(err)
} else if m["value"] != float64(200) {
t.Errorf("unexpected value: %#v", m)
}
if k, v := c.Next(); k != nil {
t.Fatalf("unexpected key/value: %#v / %#v", k, v)
}
}
