func TestParsePointsUnbalancedQuotedTags(t *testing.T) {
pts, err := models.ParsePointsString("baz,mytag=\"a x=1 1441103862125\nbaz,mytag=a z=1 1441103862126")
if err != nil {
t.Fatalf("ParsePoints failed: %v", err)
}
if exp := 2; len(pts) != exp {
t.Fatalf("ParsePoints count mismatch. got %v, exp %v", len(pts), exp)
}
// Expected " in the tag value
exp := models.MustNewPoint("baz", models.Tags{"mytag": `"a`},
models.Fields{"x": float64(1)}, time.Unix(0, 1441103862125))
if pts[0].String() != exp.String() {
t.Errorf("Point mismatch:\ngot: %v\nexp: %v", pts[0].String(), exp.String())
}
// Expected two points to ensure we did not overscan the line
exp = models.MustNewPoint("baz", models.Tags{"mytag": `a`},
models.Fields{"z": float64(1)}, time.Unix(0, 1441103862126))
if pts[1].String() != exp.String() {
t.Errorf("Point mismatch:\ngot: %v\nexp: %v", pts[1].String(), exp.String())
}
}
func TestWriteTimeTag(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
defer sh.Close()
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{}),
map[string]interface{}{"time": 1.0},
time.Unix(1, 2),
)
buf := bytes.NewBuffer(nil)
sh.SetLogOutput(buf)
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
m := index.Measurement("cpu")
if m != nil {
t.Fatal("unexpected cpu measurement")
}
pt = models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{}),
map[string]interface{}{"value": 1.0, "time": 1.0},
time.Unix(1, 2),
)
buf = bytes.NewBuffer(nil)
sh.SetLogOutput(buf)
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
m = index.Measurement("cpu")
if m == nil {
t.Fatal("expected cpu measurement")
}
if got, exp := len(m.FieldNames()), 1; got != exp {
t.Fatalf("invalid number of field names: got=%v exp=%v", got, exp)
}
}
func TestShardWriteAddNewField(t *testing.T) {
t.Skip("pending tsm1 iterator impl")
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex()
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error openeing shard: %s", err.Error())
}
defer sh.Close()
pt := models.MustNewPoint(
"cpu",
map[string]string{"host": "server"},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
pt = models.MustNewPoint(
"cpu",
map[string]string{"host": "server"},
map[string]interface{}{"value": 1.0, "value2": 2.0},
time.Unix(1, 2),
)
err = sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
if index.SeriesN() != 1 {
t.Fatalf("series wasn't in index")
}
seriesTags := index.Series(string(pt.Key())).Tags
if len(seriesTags) != len(pt.Tags()) || pt.Tags()["host"] != seriesTags["host"] {
t.Fatalf("tags weren't properly saved to series index: %v, %v", pt.Tags(), seriesTags)
}
if !reflect.DeepEqual(index.Measurement("cpu").TagKeys(), []string{"host"}) {
t.Fatalf("tag key wasn't saved to measurement index")
}
if len(index.Measurement("cpu").FieldNames()) != 2 {
t.Fatalf("field names wasn't saved to measurement index")
}
}
func TestShard_MaxTagValuesLimit(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "db", "rp", "1")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
opts.Config.MaxValuesPerTag = 1000
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
// Writing 1K series should succeed.
points := []models.Point{}
for i := 0; i < 1000; i++ {
pt := models.MustNewPoint(
"cpu",
models.Tags{{Key: []byte("host"), Value: []byte(fmt.Sprintf("server%d", i))}},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
points = append(points, pt)
}
err := sh.WritePoints(points)
if err != nil {
t.Fatalf(err.Error())
}
// Writing one more series should exceed the series limit.
pt := models.MustNewPoint(
"cpu",
models.Tags{{Key: []byte("host"), Value: []byte("server9999")}},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err = sh.WritePoints([]models.Point{pt})
if err == nil {
t.Fatal("expected error")
} else if exp, got := `max-values-per-tag limit exceeded (1000/1000): measurement="cpu" tag="host" value="server9999" dropped=1`, err.Error(); exp != got {
t.Fatalf("unexpected error message:\n\texp = %s\n\tgot = %s", exp, got)
}
sh.Close()
}
func TestMaxSeriesLimit(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
opts.Config.MaxSeriesPerDatabase = 1000
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
// Writing 1K series should succeed.
points := []models.Point{}
for i := 0; i < 1000; i++ {
pt := models.MustNewPoint(
"cpu",
map[string]string{"host": fmt.Sprintf("server%d", i)},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
points = append(points, pt)
}
err := sh.WritePoints(points)
if err != nil {
t.Fatalf(err.Error())
}
// Writing one more series should exceed the series limit.
pt := models.MustNewPoint(
"cpu",
map[string]string{"host": "server9999"},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err = sh.WritePoints([]models.Point{pt})
if err == nil {
t.Fatal("expected error")
} else if err.Error() != "max series per database exceeded: cpu,host=server9999" {
t.Fatalf("unexpected error messag:\n\texp = max series per database exceeded: cpu,host=server9999\n\tgot = %s", err.Error())
}
sh.Close()
}
// Ensure the shard writer returns an error when the server fails to accept the write.
func TestShardWriter_WriteShard_Error(t *testing.T) {
ts := newTestWriteService(writeShardFail)
s := cluster.NewService(cluster.Config{})
s.Listener = ts.muxln
s.TSDBStore = &ts.TSDBStore
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
defer ts.Close()
w := cluster.NewShardWriter(time.Minute, 1)
w.MetaClient = &metaClient{host: ts.ln.Addr().String()}
now := time.Now()
shardID := uint64(1)
ownerID := uint64(2)
var points []models.Point
points = append(points, models.MustNewPoint(
"cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
))
if err := w.WriteShard(shardID, ownerID, points); err == nil || err.Error() != "error code 1: write shard 1: failed to write" {
t.Fatalf("unexpected error: %v", err)
}
}
// benchmarkWritePoints benchmarks writing new series to a shard.
// mCnt - measurement count
// tkCnt - tag key count
// tvCnt - tag value count (values per tag)
// pntCnt - points per series. # of series = mCnt * (tvCnt ^ tkCnt)
func benchmarkWritePoints(b *testing.B, mCnt, tkCnt, tvCnt, pntCnt int) {
// Generate test series (measurements + unique tag sets).
series := genTestSeries(mCnt, tkCnt, tvCnt)
// Create index for the shard to use.
index := tsdb.NewDatabaseIndex("db")
// Generate point data to write to the shard.
points := []models.Point{}
for _, s := range series {
for val := 0.0; val < float64(pntCnt); val++ {
p := models.MustNewPoint(s.Measurement, s.Series.Tags, map[string]interface{}{"value": val}, time.Now())
points = append(points, p)
}
}
// Stop & reset timers and mem-stats before the main benchmark loop.
b.StopTimer()
b.ResetTimer()
// Run the benchmark loop.
for n := 0; n < b.N; n++ {
tmpDir, _ := ioutil.TempDir("", "shard_test")
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
shard := tsdb.NewShard(1, index, tmpShard, tmpWal, tsdb.NewEngineOptions())
shard.Open()
b.StartTimer()
// Call the function being benchmarked.
chunkedWrite(shard, points)
b.StopTimer()
shard.Close()
os.RemoveAll(tmpDir)
}
}
// Ensure the shard writer returns an error when we can't get a connection.
func TestShardWriter_Write_PoolMax(t *testing.T) {
ts := newTestWriteService(writeShardSlow)
s := cluster.NewService(cluster.Config{
ShardWriterTimeout: toml.Duration(100 * time.Millisecond),
})
s.Listener = ts.muxln
s.TSDBStore = &ts.TSDBStore
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
defer ts.Close()
w := cluster.NewShardWriter(100*time.Millisecond, 1)
w.MetaClient = &metaClient{host: ts.ln.Addr().String()}
now := time.Now()
shardID := uint64(1)
ownerID := uint64(2)
var points []models.Point
points = append(points, models.MustNewPoint(
"cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
))
go w.WriteShard(shardID, ownerID, points)
time.Sleep(time.Millisecond)
if err := w.WriteShard(shardID, ownerID, points); err == nil || err.Error() != "timed out waiting for free connection" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestFilterMatchMultipleWildcards(t *testing.T) {
p, err := graphite.NewParser([]string{
"*.* .wrong.measurement*",
"servers.* .host.measurement*", // should match this
"servers.localhost .wrong.measurement*",
"*.localhost .wrong.measurement*",
}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("cpu_load",
models.NewTags(map[string]string{"host": "server01"}),
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.server01.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
// Ensure the shard writer returns an error when dialing times out.
func TestShardWriter_Write_ErrDialTimeout(t *testing.T) {
ts := newTestWriteService(nil)
ts.TSDBStore.WriteToShardFn = ts.writeShardSuccess
s := cluster.NewService(cluster.Config{})
s.Listener = ts.muxln
s.TSDBStore = &ts.TSDBStore
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
defer ts.Close()
// Zero timeout set to support all platforms.
w := cluster.NewShardWriter(0, 1)
w.MetaClient = &metaClient{host: ts.ln.Addr().String()}
now := time.Now()
shardID := uint64(1)
ownerID := uint64(2)
var points []models.Point
points = append(points, models.MustNewPoint(
"cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
))
if err, exp := w.WriteShard(shardID, ownerID, points), "i/o timeout"; err == nil || !strings.Contains(err.Error(), exp) {
t.Fatalf("expected error %v, to contain %s", err, exp)
}
}
func TestParsePointToString(t *testing.T) {
line := `cpu,host=serverA,region=us-east bool=false,float=11,float2=12.123,int=10i,str="string val" 1000000000`
pts, err := models.ParsePoints([]byte(line))
if err != nil {
t.Fatalf(`ParsePoints() failed. got %s`, err)
}
if exp := 1; len(pts) != exp {
t.Errorf("ParsePoint() len mismatch: got %v, exp %v", len(pts), exp)
}
pt := pts[0]
got := pt.String()
if line != got {
t.Errorf("ParsePoint() to string mismatch:\n got %v\n exp %v", got, line)
}
pt = models.MustNewPoint("cpu", models.Tags{"host": "serverA", "region": "us-east"},
models.Fields{"int": 10, "float": float64(11.0), "float2": float64(12.123), "bool": false, "str": "string val"},
time.Unix(1, 0))
got = pt.String()
if line != got {
t.Errorf("NewPoint() to string mismatch:\n got %v\n exp %v", got, line)
}
}
// NewTestPoint returns a new TestPoint.
//
// NewTestPoint panics if it is not a valid models.Point.
func NewTestPoint(name string, tags models.Tags, fields models.Fields, time time.Time) TestPoint {
return TestPoint{
RawTags: tags,
RawFields: fields,
RawTime: time,
Point: models.MustNewPoint(name, tags, fields, time),
}
}
func TestNewPointUnhandledType(t *testing.T) {
// nil value
pt := models.MustNewPoint("cpu", nil, models.Fields{"value": nil}, time.Unix(0, 0))
if exp := `cpu value= 0`; pt.String() != exp {
t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp)
}
// unsupported type gets stored as string
now := time.Unix(0, 0).UTC()
pt = models.MustNewPoint("cpu", nil, models.Fields{"value": now}, time.Unix(0, 0))
if exp := `cpu value="1970-01-01 00:00:00 +0000 UTC" 0`; pt.String() != exp {
t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp)
}
if exp := "1970-01-01 00:00:00 +0000 UTC"; pt.Fields()["value"] != exp {
t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp)
}
}
func TestNewPointEscaped(t *testing.T) {
// commas
pt := models.MustNewPoint("cpu,main", models.Tags{"tag,bar": "value"}, models.Fields{"name,bar": 1.0}, time.Unix(0, 0))
if exp := `cpu\,main,tag\,bar=value name\,bar=1 0`; pt.String() != exp {
t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp)
}
// spaces
pt = models.MustNewPoint("cpu main", models.Tags{"tag bar": "value"}, models.Fields{"name bar": 1.0}, time.Unix(0, 0))
if exp := `cpu\ main,tag\ bar=value name\ bar=1 0`; pt.String() != exp {
t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp)
}
// equals
pt = models.MustNewPoint("cpu=main", models.Tags{"tag=bar": "value=foo"}, models.Fields{"name=bar": 1.0}, time.Unix(0, 0))
if exp := `cpu=main,tag\=bar=value\=foo name\=bar=1 0`; pt.String() != exp {
t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp)
}
}
func TestPrecisionString(t *testing.T) {
tags := map[string]interface{}{"value": float64(1)}
tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z")
tests := []struct {
name string
precision string
exp string
}{
{
name: "no precision",
precision: "",
exp: "cpu value=1 946730096789012345",
},
{
name: "nanosecond precision",
precision: "ns",
exp: "cpu value=1 946730096789012345",
},
{
name: "microsecond precision",
precision: "u",
exp: "cpu value=1 946730096789012",
},
{
name: "millisecond precision",
precision: "ms",
exp: "cpu value=1 946730096789",
},
{
name: "second precision",
precision: "s",
exp: "cpu value=1 946730096",
},
{
name: "minute precision",
precision: "m",
exp: "cpu value=1 15778834",
},
{
name: "hour precision",
precision: "h",
exp: "cpu value=1 262980",
},
}
for _, test := range tests {
pt := models.MustNewPoint("cpu", nil, tags, tm)
act := pt.PrecisionString(test.precision)
if act != test.exp {
t.Errorf("%s: PrecisionString() mismatch:\n actual: %v\n exp: %v",
test.name, act, test.exp)
}
}
}
func TestRoundedString(t *testing.T) {
tags := map[string]interface{}{"value": float64(1)}
tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z")
tests := []struct {
name string
precision time.Duration
exp string
}{
{
name: "no precision",
precision: time.Duration(0),
exp: "cpu value=1 946730096789012345",
},
{
name: "nanosecond precision",
precision: time.Nanosecond,
exp: "cpu value=1 946730096789012345",
},
{
name: "microsecond precision",
precision: time.Microsecond,
exp: "cpu value=1 946730096789012000",
},
{
name: "millisecond precision",
precision: time.Millisecond,
exp: "cpu value=1 946730096789000000",
},
{
name: "second precision",
precision: time.Second,
exp: "cpu value=1 946730097000000000",
},
{
name: "minute precision",
precision: time.Minute,
exp: "cpu value=1 946730100000000000",
},
{
name: "hour precision",
precision: time.Hour,
exp: "cpu value=1 946731600000000000",
},
}
for _, test := range tests {
pt := models.MustNewPoint("cpu", nil, tags, tm)
act := pt.RoundedString(test.precision)
if act != test.exp {
t.Errorf("%s: RoundedString() mismatch:\n actual: %v\n exp: %v",
test.name, act, test.exp)
}
}
}
// Ensure a point can be written via the telnet protocol.
func TestService_Telnet(t *testing.T) {
t.Parallel()
s := NewService("db0")
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
// Mock points writer.
var called int32
s.PointsWriter.WritePointsFn = func(req *cluster.WritePointsRequest) error {
atomic.StoreInt32(&called, 1)
if req.Database != "db0" {
t.Fatalf("unexpected database: %s", req.Database)
} else if req.RetentionPolicy != "" {
t.Fatalf("unexpected retention policy: %s", req.RetentionPolicy)
} else if !reflect.DeepEqual(req.Points, []models.Point{
models.MustNewPoint(
"sys.cpu.user",
map[string]string{"host": "webserver01", "cpu": "0"},
map[string]interface{}{"value": 42.5},
time.Unix(1356998400, 0),
),
}) {
spew.Dump(req.Points)
t.Fatalf("unexpected points: %#v", req.Points)
}
return nil
}
// Open connection to the service.
conn, err := net.Dial("tcp", s.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Write telnet data and close.
if _, err := conn.Write([]byte("put sys.cpu.user 1356998400 42.5 host=webserver01 cpu=0")); err != nil {
t.Fatal(err)
}
if err := conn.Close(); err != nil {
t.Fatal(err)
}
time.Sleep(10 * time.Millisecond)
// Verify that the writer was called.
if atomic.LoadInt32(&called) == 0 {
t.Fatal("points writer not called")
}
}
// Ensure the shard writer can successful write a multiple requests.
func TestShardWriter_WriteShard_Multiple(t *testing.T) {
ts := newTestWriteService(nil)
ts.TSDBStore.WriteToShardFn = ts.writeShardSuccess
s := cluster.NewService(cluster.Config{})
s.Listener = ts.muxln
s.TSDBStore = &ts.TSDBStore
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
defer ts.Close()
w := cluster.NewShardWriter(time.Minute, 1)
w.MetaClient = &metaClient{host: ts.ln.Addr().String()}
// Build a single point.
now := time.Now()
var points []models.Point
points = append(points, models.MustNewPoint("cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now))
// Write to shard twice and close.
if err := w.WriteShard(1, 2, points); err != nil {
t.Fatal(err)
} else if err := w.WriteShard(1, 2, points); err != nil {
t.Fatal(err)
} else if err := w.Close(); err != nil {
t.Fatal(err)
}
// Validate response.
responses, err := ts.ResponseN(1)
if err != nil {
t.Fatal(err)
} else if responses[0].shardID != 1 {
t.Fatalf("unexpected shard id: %d", responses[0].shardID)
}
// Validate point.
if p := responses[0].points[0]; p.Name() != "cpu" {
t.Fatalf("unexpected name: %s", p.Name())
} else if p.Fields()["value"] != int64(100) {
t.Fatalf("unexpected 'value' field: %d", p.Fields()["value"])
} else if p.Tags()["host"] != "server01" {
t.Fatalf("unexpected 'host' tag: %s", p.Tags()["host"])
} else if p.Time().UnixNano() != now.UnixNano() {
t.Fatalf("unexpected time: %s", p.Time())
}
}
// Ensure a point can be written via the HTTP protocol.
func TestService_HTTP(t *testing.T) {
t.Parallel()
s := NewService("db0")
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
// Mock points writer.
var called bool
s.PointsWriter.WritePointsFn = func(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error {
called = true
if database != "db0" {
t.Fatalf("unexpected database: %s", database)
} else if retentionPolicy != "" {
t.Fatalf("unexpected retention policy: %s", retentionPolicy)
} else if !reflect.DeepEqual(points, []models.Point{
models.MustNewPoint(
"sys.cpu.nice",
map[string]string{"dc": "lga", "host": "web01"},
map[string]interface{}{"value": 18.0},
time.Unix(1346846400, 0),
),
}) {
spew.Dump(points)
t.Fatalf("unexpected points: %#v", points)
}
return nil
}
// Write HTTP request to server.
resp, err := http.Post("http://"+s.Addr().String()+"/api/put", "application/json", strings.NewReader(`{"metric":"sys.cpu.nice", "timestamp":1346846400, "value":18, "tags":{"host":"web01", "dc":"lga"}}`))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Verify status and body.
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
// Verify that the writer was called.
if !called {
t.Fatal("points writer not called")
}
}
func benchmarkStoreOpen(b *testing.B, mCnt, tkCnt, tvCnt, pntCnt, shardCnt int) {
var path string
if err := func() error {
store := MustOpenStore()
defer store.Store.Close()
path = store.Path()
// Generate test series (measurements + unique tag sets).
series := genTestSeries(mCnt, tkCnt, tvCnt)
// Generate point data to write to the shards.
points := []models.Point{}
for _, s := range series {
for val := 0.0; val < float64(pntCnt); val++ {
p := models.MustNewPoint(s.Measurement, s.Series.Tags, map[string]interface{}{"value": val}, time.Now())
points = append(points, p)
}
}
// Create requested number of shards in the store & write points.
for shardID := 0; shardID < shardCnt; shardID++ {
if err := store.CreateShard("mydb", "myrp", uint64(shardID)); err != nil {
return fmt.Errorf("create shard: %s", err)
}
if err := store.BatchWrite(shardID, points); err != nil {
return fmt.Errorf("batch write: %s", err)
}
}
return nil
}(); err != nil {
b.Fatal(err)
}
defer os.RemoveAll(path)
// Run the benchmark loop.
b.ResetTimer()
for n := 0; n < b.N; n++ {
store := tsdb.NewStore(path)
if err := store.Open(); err != nil {
b.Fatalf("open store error: %s", err)
}
b.StopTimer()
store.Close()
b.StartTimer()
}
}
func TestShard_Disabled_WriteQuery(t *testing.T) {
sh := NewShard()
if err := sh.Open(); err != nil {
t.Fatal(err)
}
defer sh.Close()
sh.SetEnabled(false)
pt := models.MustNewPoint(
"cpu",
map[string]string{"host": "server"},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]models.Point{pt})
if err == nil {
t.Fatalf("expected shard disabled error")
}
if err != tsdb.ErrShardDisabled {
t.Fatalf(err.Error())
}
_, got := sh.CreateIterator(influxql.IteratorOptions{})
if err == nil {
t.Fatalf("expected shard disabled error")
}
if exp := tsdb.ErrShardDisabled; got != exp {
t.Fatalf("got %v, expected %v", got, exp)
}
sh.SetEnabled(true)
err = sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err = sh.CreateIterator(influxql.IteratorOptions{}); err != nil {
t.Fatalf("unexpected error: %v", got)
}
}
func TestParseNoMatch(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.*.cpu .host.measurement.cpu.measurement"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("servers.localhost.memory.VmallocChunk",
models.NewTags(map[string]string{}),
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.memory.VmallocChunk 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchWildcard(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.* .host.measurement*"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("cpu_load",
models.Tags{"host": "localhost"},
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
// Ensure the shard writer returns an error when reading times out.
func TestShardWriter_Write_ErrReadTimeout(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
w := cluster.NewShardWriter(time.Millisecond, 1)
w.MetaClient = &metaClient{host: ln.Addr().String()}
now := time.Now()
shardID := uint64(1)
ownerID := uint64(2)
var points []models.Point
points = append(points, models.MustNewPoint(
"cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
))
if err := w.WriteShard(shardID, ownerID, points); err == nil || !strings.Contains(err.Error(), "i/o timeout") {
t.Fatalf("unexpected error: %s", err)
}
}
func TestWriteTimeField(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
defer sh.Close()
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"time": "now"}),
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
buf := bytes.NewBuffer(nil)
sh.SetLogOutput(buf)
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping tag 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
key := models.MakeKey([]byte("cpu"), nil)
series := index.Series(string(key))
if series == nil {
t.Fatal("expected series")
} else if len(series.Tags) != 0 {
t.Fatalf("unexpected number of tags: got=%v exp=%v", len(series.Tags), 0)
}
}
// Ensures that when a shard is closed, it removes any series meta-data
// from the index.
func TestShard_Close_RemoveIndex(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
pt := models.MustNewPoint(
"cpu",
map[string]string{"host": "server"},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
if got, exp := index.SeriesN(), 1; got != exp {
t.Fatalf("series count mismatch: got %v, exp %v", got, exp)
}
// ensure the index gets loaded after closing and opening the shard
sh.Close()
if got, exp := index.SeriesN(), 0; got != exp {
t.Fatalf("series count mismatch: got %v, exp %v", got, exp)
}
}
func TestParseTemplateWhitespace(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement* zone=1c"}, models.NewTags(map[string]string{
"region": "us-east",
"host": "should not set",
}))
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("cpu_load",
models.NewTags(map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"}),
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchMultipleMeasurementSeparator(t *testing.T) {
p, err := graphite.NewParserWithOptions(graphite.Options{
Templates: []string{"servers.localhost .host.measurement.measurement*"},
Separator: "_",
})
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("cpu_cpu_load_10",
models.NewTags(map[string]string{"host": "localhost"}),
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu.cpu_load.10 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestShardWriteAndIndex(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
// Calling WritePoints when the engine is not open will return
// ErrEngineClosed.
if got, exp := sh.WritePoints(nil), tsdb.ErrEngineClosed; got != exp {
t.Fatalf("got %v, expected %v", got, exp)
}
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
pt := models.MustNewPoint(
"cpu",
map[string]string{"host": "server"},
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
pt.SetTime(time.Unix(2, 3))
err = sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
validateIndex := func() {
if index.SeriesN() != 1 {
t.Fatalf("series wasn't in index")
}
seriesTags := index.Series(string(pt.Key())).Tags
if len(seriesTags) != len(pt.Tags()) || pt.Tags()["host"] != seriesTags["host"] {
t.Fatalf("tags weren't properly saved to series index: %v, %v", pt.Tags(), seriesTags)
}
if !reflect.DeepEqual(index.Measurement("cpu").TagKeys(), []string{"host"}) {
t.Fatalf("tag key wasn't saved to measurement index")
}
}
validateIndex()
// ensure the index gets loaded after closing and opening the shard
sh.Close()
index = tsdb.NewDatabaseIndex("db")
sh = tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
validateIndex()
// and ensure that we can still write data
pt.SetTime(time.Unix(2, 6))
err = sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
}
// Ensure a point can be written via the telnet protocol.
func TestService_Telnet(t *testing.T) {
t.Parallel()
s := NewTestService("db0", "127.0.0.1:0")
if err := s.Service.Open(); err != nil {
t.Fatal(err)
}
defer s.Service.Close()
// Mock points writer.
var called int32
s.WritePointsFn = func(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error {
atomic.StoreInt32(&called, 1)
if database != "db0" {
t.Fatalf("unexpected database: %s", database)
} else if retentionPolicy != "" {
t.Fatalf("unexpected retention policy: %s", retentionPolicy)
} else if !reflect.DeepEqual(points, []models.Point{
models.MustNewPoint(
"sys.cpu.user",
models.NewTags(map[string]string{"host": "webserver01", "cpu": "0"}),
map[string]interface{}{"value": 42.5},
time.Unix(1356998400, 0),
),
}) {
t.Fatalf("unexpected points: %#v", points)
}
return nil
}
// Open connection to the service.
conn, err := net.Dial("tcp", s.Service.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Write telnet data and close.
if _, err := conn.Write([]byte("put sys.cpu.user 1356998400 42.5 host=webserver01 cpu=0")); err != nil {
t.Fatal(err)
}
if err := conn.Close(); err != nil {
t.Fatal(err)
}
tick := time.Tick(10 * time.Millisecond)
timeout := time.After(10 * time.Second)
for {
select {
case <-tick:
// Verify that the writer was called.
if atomic.LoadInt32(&called) > 0 {
return
}
case <-timeout:
t.Fatal("points writer not called")
}
}
}