func TestWriteTimeTag(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
defer sh.Close()
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{}),
map[string]interface{}{"time": 1.0},
time.Unix(1, 2),
)
buf := bytes.NewBuffer(nil)
sh.WithLogger(zap.New(zap.NewTextEncoder(), zap.Output(zap.AddSync(buf))))
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
m := index.Measurement("cpu")
if m != nil {
t.Fatal("unexpected cpu measurement")
}
pt = models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{}),
map[string]interface{}{"value": 1.0, "time": 1.0},
time.Unix(1, 2),
)
buf = bytes.NewBuffer(nil)
sh.WithLogger(zap.New(zap.NewTextEncoder(), zap.Output(zap.AddSync(buf))))
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
m = index.Measurement("cpu")
if m == nil {
t.Fatal("expected cpu measurement")
}
if got, exp := len(m.FieldNames()), 1; got != exp {
t.Fatalf("invalid number of field names: got=%v exp=%v", got, exp)
}
}
// NewTestService returns a new instance of Service.
func NewTestService(database string, bind string) *TestService {
s, err := NewService(Config{
BindAddress: bind,
Database: database,
ConsistencyLevel: "one",
})
if err != nil {
panic(err)
}
service := &TestService{
Service: s,
MetaClient: &internal.MetaClientMock{},
}
service.MetaClient.CreateDatabaseFn = func(db string) (*meta.DatabaseInfo, error) {
if got, exp := db, database; got != exp {
return nil, fmt.Errorf("got %v, expected %v", got, exp)
}
return nil, nil
}
if testing.Verbose() {
service.Service.WithLogger(zap.New(
zap.NewTextEncoder(),
zap.Output(os.Stderr),
))
}
service.Service.MetaClient = service.MetaClient
service.Service.PointsWriter = service
return service
}
// NewTestService returns a new *Service with default mock object members.
func NewTestService(t *testing.T) *Service {
s := NewService(NewConfig())
ms := NewMetaClient(t)
s.MetaClient = ms
s.QueryExecutor = influxql.NewQueryExecutor()
s.RunInterval = time.Millisecond
// Set Logger to write to dev/null so stdout isn't polluted.
if testing.Verbose() {
s.WithLogger(zap.New(
zap.NewTextEncoder(),
zap.Output(os.Stderr),
))
}
// Add a couple test databases and CQs.
ms.CreateDatabase("db", "rp")
ms.CreateContinuousQuery("db", "cq", `CREATE CONTINUOUS QUERY cq ON db BEGIN SELECT count(cpu) INTO cpu_count FROM cpu WHERE time > now() - 1h GROUP BY time(1s) END`)
ms.CreateDatabase("db2", "default")
ms.CreateContinuousQuery("db2", "cq2", `CREATE CONTINUOUS QUERY cq2 ON db2 BEGIN SELECT mean(value) INTO cpu_mean FROM cpu WHERE time > now() - 10m GROUP BY time(1m) END`)
ms.CreateDatabase("db3", "default")
ms.CreateContinuousQuery("db3", "cq3", `CREATE CONTINUOUS QUERY cq3 ON db3 BEGIN SELECT mean(value) INTO "1hAverages".:MEASUREMENT FROM /cpu[0-9]?/ GROUP BY time(10s) END`)
return s
}
// NewService returns an instance of the Graphite service.
func NewService(c Config) (*Service, error) {
// Use defaults where necessary.
d := c.WithDefaults()
s := Service{
bindAddress: d.BindAddress,
database: d.Database,
retentionPolicy: d.RetentionPolicy,
protocol: d.Protocol,
batchSize: d.BatchSize,
batchPending: d.BatchPending,
udpReadBuffer: d.UDPReadBuffer,
batchTimeout: time.Duration(d.BatchTimeout),
logger: zap.New(zap.NullEncoder()),
stats: &Statistics{},
defaultTags: models.StatisticTags{"proto": d.Protocol, "bind": d.BindAddress},
tcpConnections: make(map[string]*tcpConnection),
diagsKey: strings.Join([]string{"graphite", d.Protocol, d.BindAddress}, ":"),
}
parser, err := NewParserWithOptions(Options{
Templates: d.Templates,
DefaultTags: d.DefaultTags(),
Separator: d.Separator})
if err != nil {
return nil, err
}
s.parser = parser
return &s, nil
}
// NewShard returns a new initialized Shard. walPath doesn't apply to the b1 type index
func NewShard(id uint64, index *DatabaseIndex, path string, walPath string, options EngineOptions) *Shard {
db, rp := DecodeStorePath(path)
logger := zap.New(zap.NullEncoder())
s := &Shard{
index: index,
id: id,
path: path,
walPath: walPath,
options: options,
closing: make(chan struct{}),
stats: &ShardStatistics{},
defaultTags: models.StatisticTags{
"path": path,
"walPath": walPath,
"id": fmt.Sprintf("%d", id),
"database": db,
"retentionPolicy": rp,
"engine": options.EngineVersion,
},
database: db,
retentionPolicy: rp,
logger: logger,
baseLogger: logger,
EnableOnOpen: true,
}
return s
}
func NewTestService(batchSize int, batchDuration time.Duration) *TestService {
c := Config{
BindAddress: "127.0.0.1:0",
Database: "collectd_test",
BatchSize: batchSize,
BatchDuration: toml.Duration(batchDuration),
}
s := &TestService{
Config: c,
Service: NewService(c),
MetaClient: &internal.MetaClientMock{},
}
s.MetaClient.CreateDatabaseFn = func(name string) (*meta.DatabaseInfo, error) {
return nil, nil
}
s.Service.PointsWriter = s
s.Service.MetaClient = s.MetaClient
// Set the collectd types using test string.
if err := s.Service.SetTypes(typesDBText); err != nil {
panic(err)
}
if testing.Verbose() {
s.Service.WithLogger(zap.New(
zap.NewTextEncoder(),
zap.Output(os.Stderr),
))
}
return s
}
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
return &QueryExecutor{
TaskManager: NewTaskManager(),
Logger: zap.New(zap.NullEncoder()),
stats: &QueryStatistics{},
}
}
// NewService returns a configured retention policy enforcement service.
func NewService(c Config) *Service {
return &Service{
checkInterval: time.Duration(c.CheckInterval),
done: make(chan struct{}),
logger: zap.New(zap.NullEncoder()),
}
}
// NewPointsWriter returns a new instance of PointsWriter for a node.
func NewPointsWriter() *PointsWriter {
return &PointsWriter{
closing: make(chan struct{}),
WriteTimeout: DefaultWriteTimeout,
Logger: zap.New(zap.NullEncoder()),
stats: &WriteStatistics{},
}
}
// NewTaskManager creates a new TaskManager.
func NewTaskManager() *TaskManager {
return &TaskManager{
QueryTimeout: DefaultQueryTimeout,
Logger: zap.New(zap.NullEncoder()),
queries: make(map[uint64]*QueryTask),
nextID: 1,
}
}
// NewService returns an instance of the precreation service.
func NewService(c Config) (*Service, error) {
s := Service{
checkInterval: time.Duration(c.CheckInterval),
advancePeriod: time.Duration(c.AdvancePeriod),
Logger: zap.New(zap.NullEncoder()),
}
return &s, nil
}
// NewService returns a subscriber service with given settings
func NewService(c Config) *Service {
s := &Service{
Logger: zap.New(zap.NullEncoder()),
closed: true,
stats: &Statistics{},
conf: c,
}
s.NewPointsWriter = s.newPointsWriter
return s
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
return &Service{
addr: c.BindAddress,
https: c.HTTPSEnabled,
cert: c.HTTPSCertificate,
err: make(chan error),
version: c.Version,
logger: zap.New(zap.NullEncoder()),
}
}
// NewCommand return a new instance of Command.
func NewCommand() *Command {
return &Command{
closing: make(chan struct{}),
Closed: make(chan struct{}),
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
Logger: zap.New(zap.NullEncoder()),
}
}
// NewStore returns a new store with the given path and a default configuration.
// The returned store must be initialized by calling Open before using it.
func NewStore(path string) *Store {
opts := NewEngineOptions()
logger := zap.New(zap.NullEncoder())
return &Store{
path: path,
EngineOptions: opts,
Logger: logger,
baseLogger: logger,
}
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
d := *c.WithDefaults()
return &Service{
config: d,
parserChan: make(chan []byte, parserChanLen),
batcher: tsdb.NewPointBatcher(d.BatchSize, d.BatchPending, time.Duration(d.BatchTimeout)),
Logger: zap.New(zap.NullEncoder()),
stats: &Statistics{},
defaultTags: models.StatisticTags{"bind": d.BindAddress},
}
}
// NewMain return a new instance of Main.
func NewMain() *Main {
return &Main{
Logger: zap.New(
zap.NewTextEncoder(),
zap.Output(os.Stderr),
),
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// NewService returns a new instance of the collectd service.
func NewService(c Config) *Service {
s := Service{
// Use defaults where necessary.
Config: c.WithDefaults(),
Logger: zap.New(zap.NullEncoder()),
stats: &Statistics{},
defaultTags: models.StatisticTags{"bind": c.BindAddress},
}
return &s
}
// New returns a new instance of the monitor system.
func New(r Reporter, c Config) *Monitor {
return &Monitor{
globalTags: make(map[string]string),
diagRegistrations: make(map[string]diagnostics.Client),
reporter: r,
storeEnabled: c.StoreEnabled,
storeDatabase: c.StoreDatabase,
storeInterval: time.Duration(c.StoreInterval),
storeRetentionPolicy: MonitorRetentionPolicy,
Logger: zap.New(zap.NullEncoder()),
}
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
s := &Service{
Config: &c,
RunInterval: time.Duration(c.RunInterval),
RunCh: make(chan *RunRequest),
loggingEnabled: c.LogEnabled,
Logger: zap.New(zap.NullEncoder()),
stats: &Statistics{},
lastRuns: map[string]time.Time{},
}
return s
}
func NewWAL(path string) *WAL {
logger := zap.New(zap.NullEncoder())
return &WAL{
path: path,
// these options should be overriden by any options in the config
SegmentSize: DefaultSegmentSize,
closing: make(chan struct{}),
stats: &WALStatistics{},
limiter: limiter.NewFixed(defaultWaitingWALWrites),
logger: logger,
traceLogger: logger,
}
}
// NewClient returns a new *Client.
func NewClient(config *Config) *Client {
return &Client{
cacheData: &Data{
ClusterID: uint64(rand.Int63()),
Index: 1,
},
closing: make(chan struct{}),
changed: make(chan struct{}),
logger: zap.New(zap.NullEncoder()),
authCache: make(map[string]authUser, 0),
path: config.Dir,
retentionAutoCreate: config.RetentionAutoCreate,
}
}
// NewHandler returns a new instance of handler with routes.
func NewHandler(c Config) *Handler {
h := &Handler{
mux: pat.New(),
Config: &c,
Logger: zap.New(zap.NullEncoder()),
CLFLogger: log.New(os.Stderr, "[httpd] ", 0),
stats: &Statistics{},
}
h.AddRoutes([]Route{
Route{
"query-options", // Satisfy CORS checks.
"OPTIONS", "/query", false, true, h.serveOptions,
},
Route{
"query", // Query serving route.
"GET", "/query", true, true, h.serveQuery,
},
Route{
"query", // Query serving route.
"POST", "/query", true, true, h.serveQuery,
},
Route{
"write-options", // Satisfy CORS checks.
"OPTIONS", "/write", false, true, h.serveOptions,
},
Route{
"write", // Data-ingest route.
"POST", "/write", true, true, h.serveWrite,
},
Route{ // Ping
"ping",
"GET", "/ping", false, true, h.servePing,
},
Route{ // Ping
"ping-head",
"HEAD", "/ping", false, true, h.servePing,
},
Route{ // Ping w/ status
"status",
"GET", "/status", false, true, h.serveStatus,
},
Route{ // Ping w/ status
"status-head",
"HEAD", "/status", false, true, h.serveStatus,
},
}...)
return h
}
func NewFileStore(dir string) *FileStore {
logger := zap.New(zap.NullEncoder())
fs := &FileStore{
dir: dir,
lastModified: time.Time{},
logger: logger,
traceLogger: logger,
stats: &FileStoreStatistics{},
purger: &purger{
files: map[string]TSMFile{},
logger: logger,
},
}
fs.purger.fileStore = fs
return fs
}
func NewTestService(c *Config) *TestService {
if c == nil {
defaultC := NewConfig()
c = &defaultC
}
gservice, err := NewService(*c)
if err != nil {
panic(err)
}
service := &TestService{
Service: gservice,
MetaClient: &internal.MetaClientMock{},
}
service.MetaClient.CreateRetentionPolicyFn = func(string, *meta.RetentionPolicySpec, bool) (*meta.RetentionPolicyInfo, error) {
return nil, nil
}
service.MetaClient.CreateDatabaseWithRetentionPolicyFn = func(string, *meta.RetentionPolicySpec) (*meta.DatabaseInfo, error) {
return nil, nil
}
service.MetaClient.DatabaseFn = func(string) *meta.DatabaseInfo {
return nil
}
service.MetaClient.RetentionPolicyFn = func(string, string) (*meta.RetentionPolicyInfo, error) {
return nil, nil
}
if testing.Verbose() {
service.Service.WithLogger(zap.New(
zap.NewTextEncoder(),
zap.Output(os.Stderr),
))
}
// Set the Meta Client and PointsWriter.
service.Service.MetaClient = service.MetaClient
service.Service.PointsWriter = service
return service
}
// NewEngine returns a new instance of Engine.
func NewEngine(id uint64, path string, walPath string, opt tsdb.EngineOptions) tsdb.Engine {
w := NewWAL(walPath)
fs := NewFileStore(path)
cache := NewCache(uint64(opt.Config.CacheMaxMemorySize), path)
c := &Compactor{
Dir: path,
FileStore: fs,
}
logger := zap.New(zap.NullEncoder())
e := &Engine{
id: id,
path: path,
logger: logger,
traceLogger: logger,
traceLogging: opt.Config.TraceLoggingEnabled,
measurementFields: make(map[string]*tsdb.MeasurementFields),
WAL: w,
Cache: cache,
FileStore: fs,
Compactor: c,
CompactionPlan: &DefaultPlanner{
FileStore: fs,
CompactFullWriteColdDuration: time.Duration(opt.Config.CompactFullWriteColdDuration),
},
CacheFlushMemorySizeThreshold: opt.Config.CacheSnapshotMemorySize,
CacheFlushWriteColdDuration: time.Duration(opt.Config.CacheSnapshotWriteColdDuration),
enableCompactionsOnOpen: true,
stats: &EngineStatistics{},
}
if e.traceLogging {
fs.enableTraceLogging(true)
w.enableTraceLogging(true)
}
return e
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
s := &Service{
addr: c.BindAddress,
https: c.HTTPSEnabled,
cert: c.HTTPSCertificate,
key: c.HTTPSPrivateKey,
limit: c.MaxConnectionLimit,
err: make(chan error),
unixSocket: c.UnixSocketEnabled,
bindSocket: c.BindSocket,
Handler: NewHandler(c),
Logger: zap.New(zap.NullEncoder()),
}
if s.key == "" {
s.key = s.cert
}
s.Handler.Logger = s.Logger
return s
}
// NewService returns a new instance of Service.
func NewService(c Config) (*Service, error) {
// Use defaults where necessary.
d := c.WithDefaults()
s := &Service{
tls: d.TLSEnabled,
cert: d.Certificate,
BindAddress: d.BindAddress,
Database: d.Database,
RetentionPolicy: d.RetentionPolicy,
batchSize: d.BatchSize,
batchPending: d.BatchPending,
batchTimeout: time.Duration(d.BatchTimeout),
Logger: zap.New(zap.NullEncoder()),
LogPointErrors: d.LogPointErrors,
stats: &Statistics{},
defaultTags: models.StatisticTags{"bind": d.BindAddress},
}
return s, nil
}
// NewQueryExecutor returns a new instance of QueryExecutor.
// This query executor always has a node id of 0.
func NewQueryExecutor() *QueryExecutor {
e := &QueryExecutor{
QueryExecutor: influxql.NewQueryExecutor(),
}
e.StatementExecutor = &coordinator.StatementExecutor{
MetaClient: &e.MetaClient,
TSDBStore: &e.TSDBStore,
}
e.QueryExecutor.StatementExecutor = e.StatementExecutor
var out io.Writer = &e.LogOutput
if testing.Verbose() {
out = io.MultiWriter(out, os.Stderr)
}
e.QueryExecutor.WithLogger(zap.New(
zap.NewTextEncoder(),
zap.Output(os.Stderr),
))
return e
}
func TestWriteTimeField(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "shard_test")
defer os.RemoveAll(tmpDir)
tmpShard := path.Join(tmpDir, "shard")
tmpWal := path.Join(tmpDir, "wal")
index := tsdb.NewDatabaseIndex("db")
opts := tsdb.NewEngineOptions()
opts.Config.WALDir = filepath.Join(tmpDir, "wal")
sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts)
if err := sh.Open(); err != nil {
t.Fatalf("error opening shard: %s", err.Error())
}
defer sh.Close()
pt := models.MustNewPoint(
"cpu",
models.NewTags(map[string]string{"time": "now"}),
map[string]interface{}{"value": 1.0},
time.Unix(1, 2),
)
buf := bytes.NewBuffer(nil)
sh.WithLogger(zap.New(zap.NewTextEncoder(), zap.Output(zap.AddSync(buf))))
if err := sh.WritePoints([]models.Point{pt}); err != nil {
t.Fatalf("unexpected error: %v", err)
} else if got, exp := buf.String(), "dropping tag 'time'"; !strings.Contains(got, exp) {
t.Fatalf("unexpected log message: %s", strings.TrimSpace(got))
}
key := models.MakeKey([]byte("cpu"), nil)
series := index.Series(string(key))
if series == nil {
t.Fatal("expected series")
} else if len(series.Tags) != 0 {
t.Fatalf("unexpected number of tags: got=%v exp=%v", len(series.Tags), 0)
}
}