func TestParsePointsUnbalancedQuotedTags(t *testing.T) { pts, err := models.ParsePointsString("baz,mytag=\"a x=1 1441103862125\nbaz,mytag=a z=1 1441103862126") if err != nil { t.Fatalf("ParsePoints failed: %v", err) } if exp := 2; len(pts) != exp { t.Fatalf("ParsePoints count mismatch. got %v, exp %v", len(pts), exp) } // Expected " in the tag value exp := models.MustNewPoint("baz", models.Tags{"mytag": `"a`}, models.Fields{"x": float64(1)}, time.Unix(0, 1441103862125)) if pts[0].String() != exp.String() { t.Errorf("Point mismatch:\ngot: %v\nexp: %v", pts[0].String(), exp.String()) } // Expected two points to ensure we did not overscan the line exp = models.MustNewPoint("baz", models.Tags{"mytag": `a`}, models.Fields{"z": float64(1)}, time.Unix(0, 1441103862126)) if pts[1].String() != exp.String() { t.Errorf("Point mismatch:\ngot: %v\nexp: %v", pts[1].String(), exp.String()) } }
func TestWriteTimeTag(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex("db") opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } defer sh.Close() pt := models.MustNewPoint( "cpu", models.NewTags(map[string]string{}), map[string]interface{}{"time": 1.0}, time.Unix(1, 2), ) buf := bytes.NewBuffer(nil) sh.SetLogOutput(buf) if err := sh.WritePoints([]models.Point{pt}); err != nil { t.Fatalf("unexpected error: %v", err) } else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) { t.Fatalf("unexpected log message: %s", strings.TrimSpace(got)) } m := index.Measurement("cpu") if m != nil { t.Fatal("unexpected cpu measurement") } pt = models.MustNewPoint( "cpu", models.NewTags(map[string]string{}), map[string]interface{}{"value": 1.0, "time": 1.0}, time.Unix(1, 2), ) buf = bytes.NewBuffer(nil) sh.SetLogOutput(buf) if err := sh.WritePoints([]models.Point{pt}); err != nil { t.Fatalf("unexpected error: %v", err) } else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) { t.Fatalf("unexpected log message: %s", strings.TrimSpace(got)) } m = index.Measurement("cpu") if m == nil { t.Fatal("expected cpu measurement") } if got, exp := len(m.FieldNames()), 1; got != exp { t.Fatalf("invalid number of field names: got=%v exp=%v", got, exp) } }
func TestShardWriteAddNewField(t *testing.T) { t.Skip("pending tsm1 iterator impl") tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex() opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error openeing shard: %s", err.Error()) } defer sh.Close() pt := models.MustNewPoint( "cpu", map[string]string{"host": "server"}, map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) err := sh.WritePoints([]models.Point{pt}) if err != nil { t.Fatalf(err.Error()) } pt = models.MustNewPoint( "cpu", map[string]string{"host": "server"}, map[string]interface{}{"value": 1.0, "value2": 2.0}, time.Unix(1, 2), ) err = sh.WritePoints([]models.Point{pt}) if err != nil { t.Fatalf(err.Error()) } if index.SeriesN() != 1 { t.Fatalf("series wasn't in index") } seriesTags := index.Series(string(pt.Key())).Tags if len(seriesTags) != len(pt.Tags()) || pt.Tags()["host"] != seriesTags["host"] { t.Fatalf("tags weren't properly saved to series index: %v, %v", pt.Tags(), seriesTags) } if !reflect.DeepEqual(index.Measurement("cpu").TagKeys(), []string{"host"}) { t.Fatalf("tag key wasn't saved to measurement index") } if len(index.Measurement("cpu").FieldNames()) != 2 { t.Fatalf("field names wasn't saved to measurement index") } }
func TestShard_MaxTagValuesLimit(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "db", "rp", "1") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex("db") opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") opts.Config.MaxValuesPerTag = 1000 sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } // Writing 1K series should succeed. points := []models.Point{} for i := 0; i < 1000; i++ { pt := models.MustNewPoint( "cpu", models.Tags{{Key: []byte("host"), Value: []byte(fmt.Sprintf("server%d", i))}}, map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) points = append(points, pt) } err := sh.WritePoints(points) if err != nil { t.Fatalf(err.Error()) } // Writing one more series should exceed the series limit. pt := models.MustNewPoint( "cpu", models.Tags{{Key: []byte("host"), Value: []byte("server9999")}}, map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) err = sh.WritePoints([]models.Point{pt}) if err == nil { t.Fatal("expected error") } else if exp, got := `max-values-per-tag limit exceeded (1000/1000): measurement="cpu" tag="host" value="server9999" dropped=1`, err.Error(); exp != got { t.Fatalf("unexpected error message:\n\texp = %s\n\tgot = %s", exp, got) } sh.Close() }
func TestMaxSeriesLimit(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex("db") opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") opts.Config.MaxSeriesPerDatabase = 1000 sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } // Writing 1K series should succeed. points := []models.Point{} for i := 0; i < 1000; i++ { pt := models.MustNewPoint( "cpu", map[string]string{"host": fmt.Sprintf("server%d", i)}, map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) points = append(points, pt) } err := sh.WritePoints(points) if err != nil { t.Fatalf(err.Error()) } // Writing one more series should exceed the series limit. pt := models.MustNewPoint( "cpu", map[string]string{"host": "server9999"}, map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) err = sh.WritePoints([]models.Point{pt}) if err == nil { t.Fatal("expected error") } else if err.Error() != "max series per database exceeded: cpu,host=server9999" { t.Fatalf("unexpected error messag:\n\texp = max series per database exceeded: cpu,host=server9999\n\tgot = %s", err.Error()) } sh.Close() }
// Ensure the shard writer returns an error when the server fails to accept the write. func TestShardWriter_WriteShard_Error(t *testing.T) { ts := newTestWriteService(writeShardFail) s := cluster.NewService(cluster.Config{}) s.Listener = ts.muxln s.TSDBStore = &ts.TSDBStore if err := s.Open(); err != nil { t.Fatal(err) } defer s.Close() defer ts.Close() w := cluster.NewShardWriter(time.Minute, 1) w.MetaClient = &metaClient{host: ts.ln.Addr().String()} now := time.Now() shardID := uint64(1) ownerID := uint64(2) var points []models.Point points = append(points, models.MustNewPoint( "cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now, )) if err := w.WriteShard(shardID, ownerID, points); err == nil || err.Error() != "error code 1: write shard 1: failed to write" { t.Fatalf("unexpected error: %v", err) } }
// benchmarkWritePoints benchmarks writing new series to a shard. // mCnt - measurement count // tkCnt - tag key count // tvCnt - tag value count (values per tag) // pntCnt - points per series. # of series = mCnt * (tvCnt ^ tkCnt) func benchmarkWritePoints(b *testing.B, mCnt, tkCnt, tvCnt, pntCnt int) { // Generate test series (measurements + unique tag sets). series := genTestSeries(mCnt, tkCnt, tvCnt) // Create index for the shard to use. index := tsdb.NewDatabaseIndex("db") // Generate point data to write to the shard. points := []models.Point{} for _, s := range series { for val := 0.0; val < float64(pntCnt); val++ { p := models.MustNewPoint(s.Measurement, s.Series.Tags, map[string]interface{}{"value": val}, time.Now()) points = append(points, p) } } // Stop & reset timers and mem-stats before the main benchmark loop. b.StopTimer() b.ResetTimer() // Run the benchmark loop. for n := 0; n < b.N; n++ { tmpDir, _ := ioutil.TempDir("", "shard_test") tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") shard := tsdb.NewShard(1, index, tmpShard, tmpWal, tsdb.NewEngineOptions()) shard.Open() b.StartTimer() // Call the function being benchmarked. chunkedWrite(shard, points) b.StopTimer() shard.Close() os.RemoveAll(tmpDir) } }
// Ensure the shard writer returns an error when we can't get a connection. func TestShardWriter_Write_PoolMax(t *testing.T) { ts := newTestWriteService(writeShardSlow) s := cluster.NewService(cluster.Config{ ShardWriterTimeout: toml.Duration(100 * time.Millisecond), }) s.Listener = ts.muxln s.TSDBStore = &ts.TSDBStore if err := s.Open(); err != nil { t.Fatal(err) } defer s.Close() defer ts.Close() w := cluster.NewShardWriter(100*time.Millisecond, 1) w.MetaClient = &metaClient{host: ts.ln.Addr().String()} now := time.Now() shardID := uint64(1) ownerID := uint64(2) var points []models.Point points = append(points, models.MustNewPoint( "cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now, )) go w.WriteShard(shardID, ownerID, points) time.Sleep(time.Millisecond) if err := w.WriteShard(shardID, ownerID, points); err == nil || err.Error() != "timed out waiting for free connection" { t.Fatalf("unexpected error: %v", err) } }
func TestFilterMatchMultipleWildcards(t *testing.T) { p, err := graphite.NewParser([]string{ "*.* .wrong.measurement*", "servers.* .host.measurement*", // should match this "servers.localhost .wrong.measurement*", "*.localhost .wrong.measurement*", }, nil) if err != nil { t.Fatalf("unexpected error creating parser, got %v", err) } exp := models.MustNewPoint("cpu_load", models.NewTags(map[string]string{"host": "server01"}), models.Fields{"value": float64(11)}, time.Unix(1435077219, 0)) pt, err := p.Parse("servers.server01.cpu_load 11 1435077219") if err != nil { t.Fatalf("parse error: %v", err) } if exp.String() != pt.String() { t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String()) } }
// Ensure the shard writer returns an error when dialing times out. func TestShardWriter_Write_ErrDialTimeout(t *testing.T) { ts := newTestWriteService(nil) ts.TSDBStore.WriteToShardFn = ts.writeShardSuccess s := cluster.NewService(cluster.Config{}) s.Listener = ts.muxln s.TSDBStore = &ts.TSDBStore if err := s.Open(); err != nil { t.Fatal(err) } defer s.Close() defer ts.Close() // Zero timeout set to support all platforms. w := cluster.NewShardWriter(0, 1) w.MetaClient = &metaClient{host: ts.ln.Addr().String()} now := time.Now() shardID := uint64(1) ownerID := uint64(2) var points []models.Point points = append(points, models.MustNewPoint( "cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now, )) if err, exp := w.WriteShard(shardID, ownerID, points), "i/o timeout"; err == nil || !strings.Contains(err.Error(), exp) { t.Fatalf("expected error %v, to contain %s", err, exp) } }
func TestParsePointToString(t *testing.T) { line := `cpu,host=serverA,region=us-east bool=false,float=11,float2=12.123,int=10i,str="string val" 1000000000` pts, err := models.ParsePoints([]byte(line)) if err != nil { t.Fatalf(`ParsePoints() failed. got %s`, err) } if exp := 1; len(pts) != exp { t.Errorf("ParsePoint() len mismatch: got %v, exp %v", len(pts), exp) } pt := pts[0] got := pt.String() if line != got { t.Errorf("ParsePoint() to string mismatch:\n got %v\n exp %v", got, line) } pt = models.MustNewPoint("cpu", models.Tags{"host": "serverA", "region": "us-east"}, models.Fields{"int": 10, "float": float64(11.0), "float2": float64(12.123), "bool": false, "str": "string val"}, time.Unix(1, 0)) got = pt.String() if line != got { t.Errorf("NewPoint() to string mismatch:\n got %v\n exp %v", got, line) } }
// NewTestPoint returns a new TestPoint. // // NewTestPoint panics if it is not a valid models.Point. func NewTestPoint(name string, tags models.Tags, fields models.Fields, time time.Time) TestPoint { return TestPoint{ RawTags: tags, RawFields: fields, RawTime: time, Point: models.MustNewPoint(name, tags, fields, time), } }
func TestNewPointUnhandledType(t *testing.T) { // nil value pt := models.MustNewPoint("cpu", nil, models.Fields{"value": nil}, time.Unix(0, 0)) if exp := `cpu value= 0`; pt.String() != exp { t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) } // unsupported type gets stored as string now := time.Unix(0, 0).UTC() pt = models.MustNewPoint("cpu", nil, models.Fields{"value": now}, time.Unix(0, 0)) if exp := `cpu value="1970-01-01 00:00:00 +0000 UTC" 0`; pt.String() != exp { t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) } if exp := "1970-01-01 00:00:00 +0000 UTC"; pt.Fields()["value"] != exp { t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) } }
func TestNewPointEscaped(t *testing.T) { // commas pt := models.MustNewPoint("cpu,main", models.Tags{"tag,bar": "value"}, models.Fields{"name,bar": 1.0}, time.Unix(0, 0)) if exp := `cpu\,main,tag\,bar=value name\,bar=1 0`; pt.String() != exp { t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) } // spaces pt = models.MustNewPoint("cpu main", models.Tags{"tag bar": "value"}, models.Fields{"name bar": 1.0}, time.Unix(0, 0)) if exp := `cpu\ main,tag\ bar=value name\ bar=1 0`; pt.String() != exp { t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) } // equals pt = models.MustNewPoint("cpu=main", models.Tags{"tag=bar": "value=foo"}, models.Fields{"name=bar": 1.0}, time.Unix(0, 0)) if exp := `cpu=main,tag\=bar=value\=foo name\=bar=1 0`; pt.String() != exp { t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) } }
func TestPrecisionString(t *testing.T) { tags := map[string]interface{}{"value": float64(1)} tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z") tests := []struct { name string precision string exp string }{ { name: "no precision", precision: "", exp: "cpu value=1 946730096789012345", }, { name: "nanosecond precision", precision: "ns", exp: "cpu value=1 946730096789012345", }, { name: "microsecond precision", precision: "u", exp: "cpu value=1 946730096789012", }, { name: "millisecond precision", precision: "ms", exp: "cpu value=1 946730096789", }, { name: "second precision", precision: "s", exp: "cpu value=1 946730096", }, { name: "minute precision", precision: "m", exp: "cpu value=1 15778834", }, { name: "hour precision", precision: "h", exp: "cpu value=1 262980", }, } for _, test := range tests { pt := models.MustNewPoint("cpu", nil, tags, tm) act := pt.PrecisionString(test.precision) if act != test.exp { t.Errorf("%s: PrecisionString() mismatch:\n actual: %v\n exp: %v", test.name, act, test.exp) } } }
func TestRoundedString(t *testing.T) { tags := map[string]interface{}{"value": float64(1)} tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z") tests := []struct { name string precision time.Duration exp string }{ { name: "no precision", precision: time.Duration(0), exp: "cpu value=1 946730096789012345", }, { name: "nanosecond precision", precision: time.Nanosecond, exp: "cpu value=1 946730096789012345", }, { name: "microsecond precision", precision: time.Microsecond, exp: "cpu value=1 946730096789012000", }, { name: "millisecond precision", precision: time.Millisecond, exp: "cpu value=1 946730096789000000", }, { name: "second precision", precision: time.Second, exp: "cpu value=1 946730097000000000", }, { name: "minute precision", precision: time.Minute, exp: "cpu value=1 946730100000000000", }, { name: "hour precision", precision: time.Hour, exp: "cpu value=1 946731600000000000", }, } for _, test := range tests { pt := models.MustNewPoint("cpu", nil, tags, tm) act := pt.RoundedString(test.precision) if act != test.exp { t.Errorf("%s: RoundedString() mismatch:\n actual: %v\n exp: %v", test.name, act, test.exp) } } }
// Ensure a point can be written via the telnet protocol. func TestService_Telnet(t *testing.T) { t.Parallel() s := NewService("db0") if err := s.Open(); err != nil { t.Fatal(err) } defer s.Close() // Mock points writer. var called int32 s.PointsWriter.WritePointsFn = func(req *cluster.WritePointsRequest) error { atomic.StoreInt32(&called, 1) if req.Database != "db0" { t.Fatalf("unexpected database: %s", req.Database) } else if req.RetentionPolicy != "" { t.Fatalf("unexpected retention policy: %s", req.RetentionPolicy) } else if !reflect.DeepEqual(req.Points, []models.Point{ models.MustNewPoint( "sys.cpu.user", map[string]string{"host": "webserver01", "cpu": "0"}, map[string]interface{}{"value": 42.5}, time.Unix(1356998400, 0), ), }) { spew.Dump(req.Points) t.Fatalf("unexpected points: %#v", req.Points) } return nil } // Open connection to the service. conn, err := net.Dial("tcp", s.Addr().String()) if err != nil { t.Fatal(err) } defer conn.Close() // Write telnet data and close. if _, err := conn.Write([]byte("put sys.cpu.user 1356998400 42.5 host=webserver01 cpu=0")); err != nil { t.Fatal(err) } if err := conn.Close(); err != nil { t.Fatal(err) } time.Sleep(10 * time.Millisecond) // Verify that the writer was called. if atomic.LoadInt32(&called) == 0 { t.Fatal("points writer not called") } }
// Ensure the shard writer can successful write a multiple requests. func TestShardWriter_WriteShard_Multiple(t *testing.T) { ts := newTestWriteService(nil) ts.TSDBStore.WriteToShardFn = ts.writeShardSuccess s := cluster.NewService(cluster.Config{}) s.Listener = ts.muxln s.TSDBStore = &ts.TSDBStore if err := s.Open(); err != nil { t.Fatal(err) } defer s.Close() defer ts.Close() w := cluster.NewShardWriter(time.Minute, 1) w.MetaClient = &metaClient{host: ts.ln.Addr().String()} // Build a single point. now := time.Now() var points []models.Point points = append(points, models.MustNewPoint("cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now)) // Write to shard twice and close. if err := w.WriteShard(1, 2, points); err != nil { t.Fatal(err) } else if err := w.WriteShard(1, 2, points); err != nil { t.Fatal(err) } else if err := w.Close(); err != nil { t.Fatal(err) } // Validate response. responses, err := ts.ResponseN(1) if err != nil { t.Fatal(err) } else if responses[0].shardID != 1 { t.Fatalf("unexpected shard id: %d", responses[0].shardID) } // Validate point. if p := responses[0].points[0]; p.Name() != "cpu" { t.Fatalf("unexpected name: %s", p.Name()) } else if p.Fields()["value"] != int64(100) { t.Fatalf("unexpected 'value' field: %d", p.Fields()["value"]) } else if p.Tags()["host"] != "server01" { t.Fatalf("unexpected 'host' tag: %s", p.Tags()["host"]) } else if p.Time().UnixNano() != now.UnixNano() { t.Fatalf("unexpected time: %s", p.Time()) } }
// Ensure a point can be written via the HTTP protocol. func TestService_HTTP(t *testing.T) { t.Parallel() s := NewService("db0") if err := s.Open(); err != nil { t.Fatal(err) } defer s.Close() // Mock points writer. var called bool s.PointsWriter.WritePointsFn = func(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error { called = true if database != "db0" { t.Fatalf("unexpected database: %s", database) } else if retentionPolicy != "" { t.Fatalf("unexpected retention policy: %s", retentionPolicy) } else if !reflect.DeepEqual(points, []models.Point{ models.MustNewPoint( "sys.cpu.nice", map[string]string{"dc": "lga", "host": "web01"}, map[string]interface{}{"value": 18.0}, time.Unix(1346846400, 0), ), }) { spew.Dump(points) t.Fatalf("unexpected points: %#v", points) } return nil } // Write HTTP request to server. resp, err := http.Post("http://"+s.Addr().String()+"/api/put", "application/json", strings.NewReader(`{"metric":"sys.cpu.nice", "timestamp":1346846400, "value":18, "tags":{"host":"web01", "dc":"lga"}}`)) if err != nil { t.Fatal(err) } defer resp.Body.Close() // Verify status and body. if resp.StatusCode != http.StatusNoContent { t.Fatalf("unexpected status code: %d", resp.StatusCode) } // Verify that the writer was called. if !called { t.Fatal("points writer not called") } }
func benchmarkStoreOpen(b *testing.B, mCnt, tkCnt, tvCnt, pntCnt, shardCnt int) { var path string if err := func() error { store := MustOpenStore() defer store.Store.Close() path = store.Path() // Generate test series (measurements + unique tag sets). series := genTestSeries(mCnt, tkCnt, tvCnt) // Generate point data to write to the shards. points := []models.Point{} for _, s := range series { for val := 0.0; val < float64(pntCnt); val++ { p := models.MustNewPoint(s.Measurement, s.Series.Tags, map[string]interface{}{"value": val}, time.Now()) points = append(points, p) } } // Create requested number of shards in the store & write points. for shardID := 0; shardID < shardCnt; shardID++ { if err := store.CreateShard("mydb", "myrp", uint64(shardID)); err != nil { return fmt.Errorf("create shard: %s", err) } if err := store.BatchWrite(shardID, points); err != nil { return fmt.Errorf("batch write: %s", err) } } return nil }(); err != nil { b.Fatal(err) } defer os.RemoveAll(path) // Run the benchmark loop. b.ResetTimer() for n := 0; n < b.N; n++ { store := tsdb.NewStore(path) if err := store.Open(); err != nil { b.Fatalf("open store error: %s", err) } b.StopTimer() store.Close() b.StartTimer() } }
func TestShard_Disabled_WriteQuery(t *testing.T) { sh := NewShard() if err := sh.Open(); err != nil { t.Fatal(err) } defer sh.Close() sh.SetEnabled(false) pt := models.MustNewPoint( "cpu", map[string]string{"host": "server"}, map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) err := sh.WritePoints([]models.Point{pt}) if err == nil { t.Fatalf("expected shard disabled error") } if err != tsdb.ErrShardDisabled { t.Fatalf(err.Error()) } _, got := sh.CreateIterator(influxql.IteratorOptions{}) if err == nil { t.Fatalf("expected shard disabled error") } if exp := tsdb.ErrShardDisabled; got != exp { t.Fatalf("got %v, expected %v", got, exp) } sh.SetEnabled(true) err = sh.WritePoints([]models.Point{pt}) if err != nil { t.Fatalf("unexpected error: %v", err) } if _, err = sh.CreateIterator(influxql.IteratorOptions{}); err != nil { t.Fatalf("unexpected error: %v", got) } }
func TestParseNoMatch(t *testing.T) { p, err := graphite.NewParser([]string{"servers.*.cpu .host.measurement.cpu.measurement"}, nil) if err != nil { t.Fatalf("unexpected error creating parser, got %v", err) } exp := models.MustNewPoint("servers.localhost.memory.VmallocChunk", models.NewTags(map[string]string{}), models.Fields{"value": float64(11)}, time.Unix(1435077219, 0)) pt, err := p.Parse("servers.localhost.memory.VmallocChunk 11 1435077219") if err != nil { t.Fatalf("parse error: %v", err) } if exp.String() != pt.String() { t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String()) } }
func TestFilterMatchWildcard(t *testing.T) { p, err := graphite.NewParser([]string{"servers.* .host.measurement*"}, nil) if err != nil { t.Fatalf("unexpected error creating parser, got %v", err) } exp := models.MustNewPoint("cpu_load", models.Tags{"host": "localhost"}, models.Fields{"value": float64(11)}, time.Unix(1435077219, 0)) pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219") if err != nil { t.Fatalf("parse error: %v", err) } if exp.String() != pt.String() { t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String()) } }
// Ensure the shard writer returns an error when reading times out. func TestShardWriter_Write_ErrReadTimeout(t *testing.T) { ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } w := cluster.NewShardWriter(time.Millisecond, 1) w.MetaClient = &metaClient{host: ln.Addr().String()} now := time.Now() shardID := uint64(1) ownerID := uint64(2) var points []models.Point points = append(points, models.MustNewPoint( "cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now, )) if err := w.WriteShard(shardID, ownerID, points); err == nil || !strings.Contains(err.Error(), "i/o timeout") { t.Fatalf("unexpected error: %s", err) } }
func TestWriteTimeField(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex("db") opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } defer sh.Close() pt := models.MustNewPoint( "cpu", models.NewTags(map[string]string{"time": "now"}), map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) buf := bytes.NewBuffer(nil) sh.SetLogOutput(buf) if err := sh.WritePoints([]models.Point{pt}); err != nil { t.Fatalf("unexpected error: %v", err) } else if got, exp := buf.String(), "dropping tag 'time'"; !strings.Contains(got, exp) { t.Fatalf("unexpected log message: %s", strings.TrimSpace(got)) } key := models.MakeKey([]byte("cpu"), nil) series := index.Series(string(key)) if series == nil { t.Fatal("expected series") } else if len(series.Tags) != 0 { t.Fatalf("unexpected number of tags: got=%v exp=%v", len(series.Tags), 0) } }
// Ensures that when a shard is closed, it removes any series meta-data // from the index. func TestShard_Close_RemoveIndex(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex("db") opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } pt := models.MustNewPoint( "cpu", map[string]string{"host": "server"}, map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) err := sh.WritePoints([]models.Point{pt}) if err != nil { t.Fatalf(err.Error()) } if got, exp := index.SeriesN(), 1; got != exp { t.Fatalf("series count mismatch: got %v, exp %v", got, exp) } // ensure the index gets loaded after closing and opening the shard sh.Close() if got, exp := index.SeriesN(), 0; got != exp { t.Fatalf("series count mismatch: got %v, exp %v", got, exp) } }
func TestParseTemplateWhitespace(t *testing.T) { p, err := graphite.NewParser([]string{"servers.localhost .host.measurement* zone=1c"}, models.NewTags(map[string]string{ "region": "us-east", "host": "should not set", })) if err != nil { t.Fatalf("unexpected error creating parser, got %v", err) } exp := models.MustNewPoint("cpu_load", models.NewTags(map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"}), models.Fields{"value": float64(11)}, time.Unix(1435077219, 0)) pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219") if err != nil { t.Fatalf("parse error: %v", err) } if exp.String() != pt.String() { t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String()) } }
func TestFilterMatchMultipleMeasurementSeparator(t *testing.T) { p, err := graphite.NewParserWithOptions(graphite.Options{ Templates: []string{"servers.localhost .host.measurement.measurement*"}, Separator: "_", }) if err != nil { t.Fatalf("unexpected error creating parser, got %v", err) } exp := models.MustNewPoint("cpu_cpu_load_10", models.NewTags(map[string]string{"host": "localhost"}), models.Fields{"value": float64(11)}, time.Unix(1435077219, 0)) pt, err := p.Parse("servers.localhost.cpu.cpu_load.10 11 1435077219") if err != nil { t.Fatalf("parse error: %v", err) } if exp.String() != pt.String() { t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String()) } }
func TestShardWriteAndIndex(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex("db") opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) // Calling WritePoints when the engine is not open will return // ErrEngineClosed. if got, exp := sh.WritePoints(nil), tsdb.ErrEngineClosed; got != exp { t.Fatalf("got %v, expected %v", got, exp) } if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } pt := models.MustNewPoint( "cpu", map[string]string{"host": "server"}, map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) err := sh.WritePoints([]models.Point{pt}) if err != nil { t.Fatalf(err.Error()) } pt.SetTime(time.Unix(2, 3)) err = sh.WritePoints([]models.Point{pt}) if err != nil { t.Fatalf(err.Error()) } validateIndex := func() { if index.SeriesN() != 1 { t.Fatalf("series wasn't in index") } seriesTags := index.Series(string(pt.Key())).Tags if len(seriesTags) != len(pt.Tags()) || pt.Tags()["host"] != seriesTags["host"] { t.Fatalf("tags weren't properly saved to series index: %v, %v", pt.Tags(), seriesTags) } if !reflect.DeepEqual(index.Measurement("cpu").TagKeys(), []string{"host"}) { t.Fatalf("tag key wasn't saved to measurement index") } } validateIndex() // ensure the index gets loaded after closing and opening the shard sh.Close() index = tsdb.NewDatabaseIndex("db") sh = tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } validateIndex() // and ensure that we can still write data pt.SetTime(time.Unix(2, 6)) err = sh.WritePoints([]models.Point{pt}) if err != nil { t.Fatalf(err.Error()) } }
// Ensure a point can be written via the telnet protocol. func TestService_Telnet(t *testing.T) { t.Parallel() s := NewTestService("db0", "127.0.0.1:0") if err := s.Service.Open(); err != nil { t.Fatal(err) } defer s.Service.Close() // Mock points writer. var called int32 s.WritePointsFn = func(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error { atomic.StoreInt32(&called, 1) if database != "db0" { t.Fatalf("unexpected database: %s", database) } else if retentionPolicy != "" { t.Fatalf("unexpected retention policy: %s", retentionPolicy) } else if !reflect.DeepEqual(points, []models.Point{ models.MustNewPoint( "sys.cpu.user", models.NewTags(map[string]string{"host": "webserver01", "cpu": "0"}), map[string]interface{}{"value": 42.5}, time.Unix(1356998400, 0), ), }) { t.Fatalf("unexpected points: %#v", points) } return nil } // Open connection to the service. conn, err := net.Dial("tcp", s.Service.Addr().String()) if err != nil { t.Fatal(err) } defer conn.Close() // Write telnet data and close. if _, err := conn.Write([]byte("put sys.cpu.user 1356998400 42.5 host=webserver01 cpu=0")); err != nil { t.Fatal(err) } if err := conn.Close(); err != nil { t.Fatal(err) } tick := time.Tick(10 * time.Millisecond) timeout := time.After(10 * time.Second) for { select { case <-tick: // Verify that the writer was called. if atomic.LoadInt32(&called) > 0 { return } case <-timeout: t.Fatal("points writer not called") } } }