func TestWriteTimeTag(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
defer sh.Close()
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{}),
map[string]interface{}{"time": 1.0},
time.Unix(1, 2),
)
buf := bytes.NewBuffer(nil)
sh.SetLogOutput(buf)
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
m := index.Measurement("cpu")
if m != nil {
t.Fatal("unexpected cpu measurement")
}
pt = models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{}),
map[string]interface{}{"value": 1.0, "time": 1.0},
time.Unix(1, 2),
)
buf = bytes.NewBuffer(nil)
sh.SetLogOutput(buf)
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
m = index.Measurement("cpu")
if m == nil {
t.Fatal("expected cpu measurement")
}
if got, exp := len(m.FieldNames()), 1; got != exp {
t.Fatalf("invalid number of field names: got=%v exp=%v", got, exp)
}
}
func TestShardWriteAddNewField(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
defer sh.Close()
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"host": "server"}),
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
pt = models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"host": "server"}),
map[string]interface{}{"value": 1.0, "value2": 2.0},
time.Unix(1, 2),
)
err = sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
if index.SeriesN() != 1 {
t.Fatalf("series wasn't in index")
}
seriesTags := index.Series(string(pt.Key())).Tags
if len(seriesTags) != len(pt.Tags()) || pt.Tags().GetString("host") != seriesTags.GetString("host") {
t.Fatalf("tags weren't properly saved to series index: %v, %v", pt.Tags(), seriesTags)
}
if !reflect.DeepEqual(index.Measurement("cpu").TagKeys(), []string{"host"}) {
t.Fatalf("tag key wasn't saved to measurement index")
}
if len(index.Measurement("cpu").FieldNames()) != 2 {
t.Fatalf("field names wasn't saved to measurement index")
}
}
func TestFilterMatchMultipleWildcards(t *testing.T) {
p, err := graphite.NewParser([]string{
"*.* .wrong.measurement*",
"servers.* .host.measurement*", // should match this
"servers.localhost .wrong.measurement*",
"*.localhost .wrong.measurement*",
}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("cpu_load",
models.NewTags(map[string]string{"host": "server01"}),
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.server01.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
// Returns byte array of a line protocol representation of the point
func (p Point) Bytes(precision string) []byte {
key := models.MakeKey([]byte(p.Name), models.NewTags(p.Tags))
fields := models.Fields(p.Fields).MarshalBinary()
kl := len(key)
fl := len(fields)
var bytes []byte
if p.Time.IsZero() {
bytes = make([]byte, fl+kl+1)
copy(bytes, key)
bytes[kl] = ' '
copy(bytes[kl+1:], fields)
} else {
timeStr := strconv.FormatInt(p.Time.UnixNano()/models.GetPrecisionMultiplier(precision), 10)
tl := len(timeStr)
bytes = make([]byte, fl+kl+tl+2)
copy(bytes, key)
bytes[kl] = ' '
copy(bytes[kl+1:], fields)
bytes[kl+fl+1] = ' '
copy(bytes[kl+fl+2:], []byte(timeStr))
}
return bytes
}
// Ensure engine can create an iterator with a condition.
func TestEngine_CreateIterator_Condition(t *testing.T) {
t.Parallel()
e := MustOpenEngine()
defer e.Close()
e.Index().CreateMeasurementIndexIfNotExists("cpu")
e.Index().Measurement("cpu").SetFieldName("X")
e.Index().Measurement("cpu").SetFieldName("Y")
e.MeasurementFields("cpu").CreateFieldIfNotExists("value", influxql.Float, false)
e.MeasurementFields("cpu").CreateFieldIfNotExists("X", influxql.Float, false)
e.MeasurementFields("cpu").CreateFieldIfNotExists("Y", influxql.Float, false)
si := e.Index().CreateSeriesIndexIfNotExists("cpu", tsdb.NewSeries("cpu,host=A", models.NewTags(map[string]string{"host": "A"})))
si.AssignShard(1)
if err := e.WritePointsString(
`cpu,host=A value=1.1 1000000000`,
`cpu,host=A X=10 1000000000`,
`cpu,host=A Y=100 1000000000`,
`cpu,host=A value=1.2 2000000000`,
`cpu,host=A value=1.3 3000000000`,
`cpu,host=A X=20 3000000000`,
`cpu,host=A Y=200 3000000000`,
); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
itr, err := e.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Dimensions: []string{"host"},
Condition: influxql.MustParseExpr(`X = 10 OR Y > 150`),
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
Ascending: true,
})
if err != nil {
t.Fatal(err)
}
fitr := itr.(influxql.FloatIterator)
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(0): %v", err)
} else if !reflect.DeepEqual(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1000000000, Value: 1.1}) {
t.Fatalf("unexpected point(0): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected point(1): %v", err)
} else if !reflect.DeepEqual(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 3000000000, Value: 1.3}) {
t.Fatalf("unexpected point(1): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("expected eof, got error: %v", err)
} else if p != nil {
t.Fatalf("expected eof: %v", p)
}
}
// DefaultTags returns the config's tags.
func (c *Config) DefaultTags() models.Tags {
m := make(map[string]string, len(c.Tags))
for _, t := range c.Tags {
parts := strings.Split(t, "=")
m[parts[0]] = parts[1]
}
return models.NewTags(m)
}
// MustInitBenchmarkEngine creates a new engine and fills it with points.
// Reuses previous engine if the same parameters were used.
func MustInitBenchmarkEngine(pointN int) *Engine {
// Reuse engine, if available.
if benchmark.Engine != nil {
if benchmark.PointN == pointN {
return benchmark.Engine
}
// Otherwise close and remove it.
benchmark.Engine.Close()
benchmark.Engine = nil
}
const batchSize = 1000
if pointN%batchSize != 0 {
panic(fmt.Sprintf("point count (%d) must be a multiple of batch size (%d)", pointN, batchSize))
}
e := MustOpenEngine()
// Initialize metadata.
e.Index().CreateMeasurementIndexIfNotExists("cpu")
e.MeasurementFields("cpu").CreateFieldIfNotExists("value", influxql.Float, false)
si := e.Index().CreateSeriesIndexIfNotExists("cpu", tsdb.NewSeries("cpu,host=A", models.NewTags(map[string]string{"host": "A"})))
si.AssignShard(1)
// Generate time ascending points with jitterred time & value.
rand := rand.New(rand.NewSource(0))
for i := 0; i < pointN; i += batchSize {
var buf bytes.Buffer
for j := 0; j < batchSize; j++ {
fmt.Fprintf(&buf, "cpu,host=%s value=%d %d",
hostNames[j%len(hostNames)],
100+rand.Intn(50)-25,
(time.Duration(i+j)*time.Second)+(time.Duration(rand.Intn(500)-250)*time.Millisecond),
)
if j != pointN-1 {
fmt.Fprint(&buf, "\n")
}
}
if err := e.WritePointsString(buf.String()); err != nil {
panic(err)
}
}
if err := e.WriteSnapshot(); err != nil {
panic(err)
}
// Force garbage collection.
runtime.GC()
// Save engine reference for reuse.
benchmark.Engine = e
benchmark.PointN = pointN
return e
}
// Ensure engine can create an iterator with auxilary fields.
func TestEngine_CreateIterator_Aux(t *testing.T) {
t.Parallel()
e := MustOpenEngine()
defer e.Close()
e.Index().CreateMeasurementIndexIfNotExists("cpu")
e.MeasurementFields("cpu").CreateFieldIfNotExists("value", influxql.Float, false)
e.MeasurementFields("cpu").CreateFieldIfNotExists("F", influxql.Float, false)
si := e.Index().CreateSeriesIndexIfNotExists("cpu", tsdb.NewSeries("cpu,host=A", models.NewTags(map[string]string{"host": "A"})))
si.AssignShard(1)
if err := e.WritePointsString(
`cpu,host=A value=1.1 1000000000`,
`cpu,host=A F=100 1000000000`,
`cpu,host=A value=1.2 2000000000`,
`cpu,host=A value=1.3 3000000000`,
`cpu,host=A F=200 3000000000`,
); err != nil {
t.Fatalf("failed to write points: %s", err.Error())
}
itr, err := e.CreateIterator(influxql.IteratorOptions{
Expr: influxql.MustParseExpr(`value`),
Aux: []influxql.VarRef{{Val: "F"}},
Dimensions: []string{"host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
StartTime: influxql.MinTime,
EndTime: influxql.MaxTime,
Ascending: true,
})
if err != nil {
t.Fatal(err)
}
fitr := itr.(influxql.FloatIterator)
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(0): %v", err)
} else if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1000000000, Value: 1.1, Aux: []interface{}{float64(100)}}) {
t.Fatalf("unexpected point(0): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(1): %v", err)
} else if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 2000000000, Value: 1.2, Aux: []interface{}{(*float64)(nil)}}) {
t.Fatalf("unexpected point(1): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("unexpected error(2): %v", err)
} else if !deep.Equal(p, &influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 3000000000, Value: 1.3, Aux: []interface{}{float64(200)}}) {
t.Fatalf("unexpected point(2): %v", p)
}
if p, err := fitr.Next(); err != nil {
t.Fatalf("expected eof, got error: %v", err)
} else if p != nil {
t.Fatalf("expected eof: %v", p)
}
}
// AddPoint adds a point to the WritePointRequest with field key 'value'
func (w *WritePointsRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string) {
pt, err := models.NewPoint(
name, models.NewTags(tags), map[string]interface{}{"value": value}, timestamp,
)
if err != nil {
return
}
w.Points = append(w.Points, pt)
}
func Test_Service_UDP(t *testing.T) {
t.Parallel()
now := time.Now().UTC().Round(time.Second)
config := Config{}
config.Database = "graphitedb"
config.BatchSize = 0 // No batching.
config.BatchTimeout = toml.Duration(time.Second)
config.BindAddress = ":10000"
config.Protocol = "udp"
service := NewTestService(&config)
// Allow test to wait until points are written.
var wg sync.WaitGroup
wg.Add(1)
service.WritePointsFn = func(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error {
defer wg.Done()
pt, _ := models.NewPoint(
"cpu",
models.NewTags(map[string]string{}),
map[string]interface{}{"value": 23.456},
time.Unix(now.Unix(), 0))
if database != "graphitedb" {
t.Fatalf("unexpected database: %s", database)
} else if retentionPolicy != "" {
t.Fatalf("unexpected retention policy: %s", retentionPolicy)
} else if points[0].String() != pt.String() {
t.Fatalf("unexpected points: %#v", points[0].String())
}
return nil
}
if err := service.Service.Open(); err != nil {
t.Fatalf("failed to open Graphite service: %s", err.Error())
}
// Connect to the graphite endpoint we just spun up
_, port, _ := net.SplitHostPort(service.Service.Addr().String())
conn, err := net.Dial("udp", "127.0.0.1:"+port)
if err != nil {
t.Fatal(err)
}
data := []byte(`cpu 23.456 `)
data = append(data, []byte(fmt.Sprintf("%d", now.Unix()))...)
data = append(data, '\n')
_, err = conn.Write(data)
if err != nil {
t.Fatal(err)
}
wg.Wait()
conn.Close()
}
// storeStatistics writes the statistics to an InfluxDB system.
func (m *Monitor) storeStatistics() {
defer m.wg.Done()
m.Logger.Printf("Storing statistics in database '%s' retention policy '%s', at interval %s",
m.storeDatabase, m.storeRetentionPolicy, m.storeInterval)
hostname, _ := os.Hostname()
m.SetGlobalTag("hostname", hostname)
// Wait until an even interval to start recording monitor statistics.
// If we are interrupted before the interval for some reason, exit early.
if err := m.waitUntilInterval(m.storeInterval); err != nil {
return
}
tick := time.NewTicker(m.storeInterval)
defer tick.Stop()
for {
select {
case now := <-tick.C:
now = now.Truncate(m.storeInterval)
func() {
m.mu.Lock()
defer m.mu.Unlock()
m.createInternalStorage()
}()
stats, err := m.Statistics(m.globalTags)
if err != nil {
m.Logger.Printf("failed to retrieve registered statistics: %s", err)
return
}
points := make(models.Points, 0, len(stats))
for _, s := range stats {
pt, err := models.NewPoint(s.Name, models.NewTags(s.Tags), s.Values, now)
if err != nil {
m.Logger.Printf("Dropping point %v: %v", s.Name, err)
return
}
points = append(points, pt)
}
func() {
m.mu.RLock()
defer m.mu.RUnlock()
if err := m.PointsWriter.WritePoints(m.storeDatabase, m.storeRetentionPolicy, points); err != nil {
m.Logger.Printf("failed to store statistics: %s", err)
}
}()
case <-m.done:
m.Logger.Printf("terminating storage of statistics")
return
}
}
}
// MarshalString renders string representation of a Point with specified
// precision. The default precision is nanoseconds.
func (p *Point) MarshalString() string {
pt, err := models.NewPoint(p.Measurement, models.NewTags(p.Tags), p.Fields, p.Time)
if err != nil {
return "# ERROR: " + err.Error() + " " + p.Measurement
}
if p.Precision == "" || p.Precision == "ns" || p.Precision == "n" {
return pt.String()
}
return pt.PrecisionString(p.Precision)
}
// NewGaugeMetric returns a gauge metric.
// Gauge metrics should be used when the metric is can arbitrarily go up and
// down. ie, temperature, memory usage, cpu usage, etc.
func NewGaugeMetric(
name string,
tags map[string]string,
fields map[string]interface{},
t time.Time,
) (Metric, error) {
pt, err := models.NewPoint(name, models.NewTags(tags), fields, t)
if err != nil {
return nil, err
}
return &metric{
pt: pt,
mType: Gauge,
}, nil
}
func TestParseDefaultTags(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement*"}, models.NewTags(map[string]string{
"region": "us-east",
"zone": "1c",
"host": "should not set",
}))
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("cpu_load",
models.NewTags(map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"}),
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
// Unmarshal translates a collectd packet into InfluxDB data points.
func (s *Service) UnmarshalCollectd(packet *gollectd.Packet) []models.Point {
// Prefer high resolution timestamp.
var timestamp time.Time
if packet.TimeHR > 0 {
// TimeHR is "near" nanosecond measurement, but not exactly nanasecond time
// Since we store time in microseconds, we round here (mostly so tests will work easier)
sec := packet.TimeHR >> 30
// Shifting, masking, and dividing by 1 billion to get nanoseconds.
nsec := ((packet.TimeHR & 0x3FFFFFFF) << 30) / 1000 / 1000 / 1000
timestamp = time.Unix(int64(sec), int64(nsec)).UTC().Round(time.Microsecond)
} else {
// If we don't have high resolution time, fall back to basic unix time
timestamp = time.Unix(int64(packet.Time), 0).UTC()
}
var points []models.Point
for i := range packet.Values {
name := fmt.Sprintf("%s_%s", packet.Plugin, packet.Values[i].Name)
tags := make(map[string]string)
fields := make(map[string]interface{})
fields["value"] = packet.Values[i].Value
if packet.Hostname != "" {
tags["host"] = packet.Hostname
}
if packet.PluginInstance != "" {
tags["instance"] = packet.PluginInstance
}
if packet.Type != "" {
tags["type"] = packet.Type
}
if packet.TypeInstance != "" {
tags["type_instance"] = packet.TypeInstance
}
// Drop invalid points
p, err := models.NewPoint(name, models.NewTags(tags), fields, timestamp)
if err != nil {
s.Logger.Printf("Dropping point %v: %v", name, err)
atomic.AddInt64(&s.stats.InvalidDroppedPoints, 1)
continue
}
points = append(points, p)
}
return points
}
// Ensure a point can be written via the HTTP protocol.
func TestService_HTTP(t *testing.T) {
t.Parallel()
s := NewTestService("db0", "127.0.0.1:0")
if err := s.Service.Open(); err != nil {
t.Fatal(err)
}
defer s.Service.Close()
// Mock points writer.
var called bool
s.WritePointsFn = func(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error {
called = true
if database != "db0" {
t.Fatalf("unexpected database: %s", database)
} else if retentionPolicy != "" {
t.Fatalf("unexpected retention policy: %s", retentionPolicy)
} else if !reflect.DeepEqual(points, []models.Point{
models.MustNewPoint(
"sys.cpu.nice",
models.NewTags(map[string]string{"dc": "lga", "host": "web01"}),
map[string]interface{}{"value": 18.0},
time.Unix(1346846400, 0),
),
}) {
spew.Dump(points)
t.Fatalf("unexpected points: %#v", points)
}
return nil
}
// Write HTTP request to server.
resp, err := http.Post("http://"+s.Service.Addr().String()+"/api/put", "application/json", strings.NewReader(`{"metric":"sys.cpu.nice", "timestamp":1346846400, "value":18, "tags":{"host":"web01", "dc":"lga"}}`))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Verify status and body.
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
// Verify that the writer was called.
if !called {
t.Fatal("points writer not called")
}
}
// Unmarshal translates a ValueList into InfluxDB data points.
func (s *Service) UnmarshalValueList(vl *api.ValueList) []models.Point {
timestamp := vl.Time.UTC()
var points []models.Point
for i := range vl.Values {
var name string
name = fmt.Sprintf("%s_%s", vl.Identifier.Plugin, vl.DSName(i))
tags := make(map[string]string)
fields := make(map[string]interface{})
// Convert interface back to actual type, then to float64
switch value := vl.Values[i].(type) {
case api.Gauge:
fields["value"] = float64(value)
case api.Derive:
fields["value"] = float64(value)
case api.Counter:
fields["value"] = float64(value)
}
if vl.Identifier.Host != "" {
tags["host"] = vl.Identifier.Host
}
if vl.Identifier.PluginInstance != "" {
tags["instance"] = vl.Identifier.PluginInstance
}
if vl.Identifier.Type != "" {
tags["type"] = vl.Identifier.Type
}
if vl.Identifier.TypeInstance != "" {
tags["type_instance"] = vl.Identifier.TypeInstance
}
// Drop invalid points
p, err := models.NewPoint(name, models.NewTags(tags), fields, timestamp)
if err != nil {
s.Logger.Info(fmt.Sprintf("Dropping point %v: %v", name, err))
atomic.AddInt64(&s.stats.InvalidDroppedPoints, 1)
continue
}
points = append(points, p)
}
return points
}
func TestShard_Disabled_WriteQuery(t *testing.T) {
sh := NewShard()
if err := sh.Open(); err != nil {
t.Fatal(err)
}
defer sh.Close()
sh.SetEnabled(false)
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"host": "server"}),
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]models.Point{pt})
if err == nil {
t.Fatalf("expected shard disabled error")
}
if err != tsdb.ErrShardDisabled {
t.Fatalf(err.Error())
}
_, got := sh.CreateIterator(influxql.IteratorOptions{})
if err == nil {
t.Fatalf("expected shard disabled error")
}
if exp := tsdb.ErrShardDisabled; got != exp {
t.Fatalf("got %v, expected %v", got, exp)
}
sh.SetEnabled(true)
err = sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err = sh.CreateIterator(influxql.IteratorOptions{}); err != nil {
t.Fatalf("unexpected error: %v", got)
}
}
// NewPoint returns a point with the given timestamp. If a timestamp is not
// given, then data is sent to the database without a timestamp, in which case
// the server will assign local time upon reception. NOTE: it is recommended to
// send data with a timestamp.
func NewPoint(
name string,
tags map[string]string,
fields map[string]interface{},
t ...time.Time,
) (*Point, error) {
var T time.Time
if len(t) > 0 {
T = t[0]
}
pt, err := models.NewPoint(name, models.NewTags(tags), fields, T)
if err != nil {
return nil, err
}
return &Point{
pt: pt,
}, nil
}
func TestParseNoMatch(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.*.cpu .host.measurement.cpu.measurement"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("servers.localhost.memory.VmallocChunk",
models.NewTags(map[string]string{}),
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.memory.VmallocChunk 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestWriteTimeField(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
defer sh.Close()
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"time": "now"}),
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
buf := bytes.NewBuffer(nil)
sh.SetLogOutput(buf)
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping tag 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
key := models.MakeKey([]byte("cpu"), nil)
series := index.Series(string(key))
if series == nil {
t.Fatal("expected series")
} else if len(series.Tags) != 0 {
t.Fatalf("unexpected number of tags: got=%v exp=%v", len(series.Tags), 0)
}
}
// Ensures that when a shard is closed, it removes any series meta-data
// from the index.
func TestShard_Close_RemoveIndex(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"host": "server"}),
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
err := sh.WritePoints([]models.Point{pt})
if err != nil {
t.Fatalf(err.Error())
}
if got, exp := index.SeriesN(), 1; got != exp {
t.Fatalf("series count mismatch: got %v, exp %v", got, exp)
}
// ensure the index gets loaded after closing and opening the shard
sh.Close()
if got, exp := index.SeriesN(), 0; got != exp {
t.Fatalf("series count mismatch: got %v, exp %v", got, exp)
}
}
func TestFilterMatchMultipleMeasurementSeparator(t *testing.T) {
p, err := graphite.NewParserWithOptions(graphite.Options{
Templates: []string{"servers.localhost .host.measurement.measurement*"},
Separator: "_",
})
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := models.MustNewPoint("cpu_cpu_load_10",
models.NewTags(map[string]string{"host": "localhost"}),
models.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu.cpu_load.10 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
// convertRowToPoints will convert a query result Row into Points that can be written back in.
func convertRowToPoints(measurementName string, row *models.Row) ([]models.Point, error) {
// figure out which parts of the result are the time and which are the fields
timeIndex := -1
fieldIndexes := make(map[string]int)
for i, c := range row.Columns {
if c == "time" {
timeIndex = i
} else {
fieldIndexes[c] = i
}
}
if timeIndex == -1 {
return nil, errors.New("error finding time index in result")
}
points := make([]models.Point, 0, len(row.Values))
for _, v := range row.Values {
vals := make(map[string]interface{})
for fieldName, fieldIndex := range fieldIndexes {
val := v[fieldIndex]
if val != nil {
vals[fieldName] = v[fieldIndex]
}
}
p, err := models.NewPoint(measurementName, models.NewTags(row.Tags), vals, v[timeIndex].(time.Time))
if err != nil {
// Drop points that can't be stored
continue
}
points = append(points, p)
}
return points, nil
}
// Tests concurrently writing to the same shard with different field types which
// can trigger a panic when the shard is snapshotted to TSM files.
func TestShard_WritePoints_FieldConflictConcurrent(t *testing.T) {
if testing.Short() {
t.Skip()
}
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
defer sh.Close()
points := make([]models.Point, 0, 1000)
for i := 0; i < cap(points); i++ {
if i < 500 {
points = append(points, models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"host": "server"}),
map[string]interface{}{"value": 1.0},
time.Unix(int64(i), 0),
))
} else {
points = append(points, models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"host": "server"}),
map[string]interface{}{"value": int64(1)},
time.Unix(int64(i), 0),
))
}
}
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for i := 0; i < 50; i++ {
if err := sh.DeleteMeasurement("cpu", []string{"cpu,host=server"}); err != nil {
t.Fatalf(err.Error())
}
_ = sh.WritePoints(points[:500])
if f, err := sh.CreateSnapshot(); err == nil {
os.RemoveAll(f)
}
}
}()
go func() {
defer wg.Done()
for i := 0; i < 50; i++ {
if err := sh.DeleteMeasurement("cpu", []string{"cpu,host=server"}); err != nil {
t.Fatalf(err.Error())
}
_ = sh.WritePoints(points[500:])
if f, err := sh.CreateSnapshot(); err == nil {
os.RemoveAll(f)
}
}
}()
wg.Wait()
}
func genTestSeries(mCnt, tCnt, vCnt int) []*TestSeries {
measurements := genStrList("measurement", mCnt)
tagSets := NewTagSetGenerator(tCnt, vCnt).AllSets()
series := []*TestSeries{}
for _, m := range measurements {
for _, ts := range tagSets {
series = append(series, &TestSeries{
Measurement: m,
Series: tsdb.NewSeries(fmt.Sprintf("%s:%s", m, string(tsdb.MarshalTags(ts))), models.NewTags(ts)),
})
}
}
return series
}
// ServeHTTP implements OpenTSDB's HTTP /api/put endpoint
func (h *Handler) servePut(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
// Require POST method.
if r.Method != "POST" {
http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
return
}
// Wrap reader if it's gzip encoded.
var br *bufio.Reader
if r.Header.Get("Content-Encoding") == "gzip" {
zr, err := gzip.NewReader(r.Body)
if err != nil {
http.Error(w, "could not read gzip, "+err.Error(), http.StatusBadRequest)
return
}
br = bufio.NewReader(zr)
} else {
br = bufio.NewReader(r.Body)
}
// Lookahead at the first byte.
f, err := br.Peek(1)
if err != nil || len(f) != 1 {
http.Error(w, "peek error: "+err.Error(), http.StatusBadRequest)
return
}
// Peek to see if this is a JSON array.
var multi bool
switch f[0] {
case '{':
case '[':
multi = true
default:
http.Error(w, "expected JSON array or hash", http.StatusBadRequest)
return
}
// Decode JSON data into slice of points.
dps := make([]point, 1)
if dec := json.NewDecoder(br); multi {
if err = dec.Decode(&dps); err != nil {
http.Error(w, "json array decode error", http.StatusBadRequest)
return
}
} else {
if err = dec.Decode(&dps[0]); err != nil {
http.Error(w, "json object decode error", http.StatusBadRequest)
return
}
}
// Convert points into TSDB points.
points := make([]models.Point, 0, len(dps))
for i := range dps {
p := dps[i]
// Convert timestamp to Go time.
// If time value is over a billion then it's microseconds.
var ts time.Time
if p.Time < 10000000000 {
ts = time.Unix(p.Time, 0)
} else {
ts = time.Unix(p.Time/1000, (p.Time%1000)*1000)
}
pt, err := models.NewPoint(p.Metric, models.NewTags(p.Tags), map[string]interface{}{"value": p.Value}, ts)
if err != nil {
h.Logger.Printf("Dropping point %v: %v", p.Metric, err)
if h.stats != nil {
atomic.AddInt64(&h.stats.InvalidDroppedPoints, 1)
}
continue
}
points = append(points, pt)
}
// Write points.
if err := h.PointsWriter.WritePoints(h.Database, h.RetentionPolicy, models.ConsistencyLevelAny, points); influxdb.IsClientError(err) {
h.Logger.Println("write series error: ", err)
http.Error(w, "write series error: "+err.Error(), http.StatusBadRequest)
return
} else if err != nil {
h.Logger.Println("write series error: ", err)
http.Error(w, "write series error: "+err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
// Parse performs Graphite parsing of a single line.
func (p *Parser) Parse(line string) (models.Point, error) {
// Break into 3 fields (name, value, timestamp).
fields := strings.Fields(line)
if len(fields) != 2 && len(fields) != 3 {
return nil, fmt.Errorf("received %q which doesn't have required fields", line)
}
// decode the name and tags
template := p.matcher.Match(fields[0])
measurement, tags, field, err := template.Apply(fields[0])
if err != nil {
return nil, err
}
// Could not extract measurement, use the raw value
if measurement == "" {
measurement = fields[0]
}
// Parse value.
v, err := strconv.ParseFloat(fields[1], 64)
if err != nil {
return nil, fmt.Errorf(`field "%s" value: %s`, fields[0], err)
}
if math.IsNaN(v) || math.IsInf(v, 0) {
return nil, &UnsupportedValueError{Field: fields[0], Value: v}
}
fieldValues := map[string]interface{}{}
if field != "" {
fieldValues[field] = v
} else {
fieldValues["value"] = v
}
// If no 3rd field, use now as timestamp
timestamp := time.Now().UTC()
if len(fields) == 3 {
// Parse timestamp.
unixTime, err := strconv.ParseFloat(fields[2], 64)
if err != nil {
return nil, fmt.Errorf(`field "%s" time: %s`, fields[0], err)
}
// -1 is a special value that gets converted to current UTC time
// See https://github.com/graphite-project/carbon/issues/54
if unixTime != float64(-1) {
// Check if we have fractional seconds
timestamp = time.Unix(int64(unixTime), int64((unixTime-math.Floor(unixTime))*float64(time.Second)))
if timestamp.Before(MinDate) || timestamp.After(MaxDate) {
return nil, fmt.Errorf("timestamp out of range")
}
}
}
// Set the default tags on the point if they are not already set
for _, t := range p.tags {
if _, ok := tags[string(t.Key)]; !ok {
tags[string(t.Key)] = string(t.Value)
}
}
return models.NewPoint(measurement, models.NewTags(tags), fieldValues, timestamp)
}
// Statistics returns statistics for periodic monitoring.
func (s *Service) Statistics(tags map[string]string) []models.Statistic {
return s.Handler.Statistics(models.NewTags(map[string]string{"bind": s.addr}).Merge(tags).Map())
}
// Ensure a point can be written via the telnet protocol.
func TestService_Telnet(t *testing.T) {
t.Parallel()
s := NewTestService("db0", "127.0.0.1:0")
if err := s.Service.Open(); err != nil {
t.Fatal(err)
}
defer s.Service.Close()
// Mock points writer.
var called int32
s.WritePointsFn = func(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error {
atomic.StoreInt32(&called, 1)
if database != "db0" {
t.Fatalf("unexpected database: %s", database)
} else if retentionPolicy != "" {
t.Fatalf("unexpected retention policy: %s", retentionPolicy)
} else if !reflect.DeepEqual(points, []models.Point{
models.MustNewPoint(
"sys.cpu.user",
models.NewTags(map[string]string{"host": "webserver01", "cpu": "0"}),
map[string]interface{}{"value": 42.5},
time.Unix(1356998400, 0),
),
}) {
t.Fatalf("unexpected points: %#v", points)
}
return nil
}
// Open connection to the service.
conn, err := net.Dial("tcp", s.Service.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Write telnet data and close.
if _, err := conn.Write([]byte("put sys.cpu.user 1356998400 42.5 host=webserver01 cpu=0")); err != nil {
t.Fatal(err)
}
if err := conn.Close(); err != nil {
t.Fatal(err)
}
tick := time.Tick(10 * time.Millisecond)
timeout := time.After(10 * time.Second)
for {
select {
case <-tick:
// Verify that the writer was called.
if atomic.LoadInt32(&called) > 0 {
return
}
case <-timeout:
t.Fatal("points writer not called")
}
}
}