func TestWriteTimeTag(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex("db") opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } defer sh.Close() pt := models.MustNewPoint( "cpu", models.NewTags(map[string]string{}), map[string]interface{}{"time": 1.0}, time.Unix(1, 2), ) buf := bytes.NewBuffer(nil) sh.WithLogger(zap.New(zap.NewTextEncoder(), zap.Output(zap.AddSync(buf)))) if err := sh.WritePoints([]models.Point{pt}); err != nil { t.Fatalf("unexpected error: %v", err) } else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) { t.Fatalf("unexpected log message: %s", strings.TrimSpace(got)) } m := index.Measurement("cpu") if m != nil { t.Fatal("unexpected cpu measurement") } pt = models.MustNewPoint( "cpu", models.NewTags(map[string]string{}), map[string]interface{}{"value": 1.0, "time": 1.0}, time.Unix(1, 2), ) buf = bytes.NewBuffer(nil) sh.WithLogger(zap.New(zap.NewTextEncoder(), zap.Output(zap.AddSync(buf)))) if err := sh.WritePoints([]models.Point{pt}); err != nil { t.Fatalf("unexpected error: %v", err) } else if got, exp := buf.String(), "dropping field 'time'"; !strings.Contains(got, exp) { t.Fatalf("unexpected log message: %s", strings.TrimSpace(got)) } m = index.Measurement("cpu") if m == nil { t.Fatal("expected cpu measurement") } if got, exp := len(m.FieldNames()), 1; got != exp { t.Fatalf("invalid number of field names: got=%v exp=%v", got, exp) } }
// NewTestService returns a new instance of Service. func NewTestService(database string, bind string) *TestService { s, err := NewService(Config{ BindAddress: bind, Database: database, ConsistencyLevel: "one", }) if err != nil { panic(err) } service := &TestService{ Service: s, MetaClient: &internal.MetaClientMock{}, } service.MetaClient.CreateDatabaseFn = func(db string) (*meta.DatabaseInfo, error) { if got, exp := db, database; got != exp { return nil, fmt.Errorf("got %v, expected %v", got, exp) } return nil, nil } if testing.Verbose() { service.Service.WithLogger(zap.New( zap.NewTextEncoder(), zap.Output(os.Stderr), )) } service.Service.MetaClient = service.MetaClient service.Service.PointsWriter = service return service }
// NewTestService returns a new *Service with default mock object members. func NewTestService(t *testing.T) *Service { s := NewService(NewConfig()) ms := NewMetaClient(t) s.MetaClient = ms s.QueryExecutor = influxql.NewQueryExecutor() s.RunInterval = time.Millisecond // Set Logger to write to dev/null so stdout isn't polluted. if testing.Verbose() { s.WithLogger(zap.New( zap.NewTextEncoder(), zap.Output(os.Stderr), )) } // Add a couple test databases and CQs. ms.CreateDatabase("db", "rp") ms.CreateContinuousQuery("db", "cq", `CREATE CONTINUOUS QUERY cq ON db BEGIN SELECT count(cpu) INTO cpu_count FROM cpu WHERE time > now() - 1h GROUP BY time(1s) END`) ms.CreateDatabase("db2", "default") ms.CreateContinuousQuery("db2", "cq2", `CREATE CONTINUOUS QUERY cq2 ON db2 BEGIN SELECT mean(value) INTO cpu_mean FROM cpu WHERE time > now() - 10m GROUP BY time(1m) END`) ms.CreateDatabase("db3", "default") ms.CreateContinuousQuery("db3", "cq3", `CREATE CONTINUOUS QUERY cq3 ON db3 BEGIN SELECT mean(value) INTO "1hAverages".:MEASUREMENT FROM /cpu[0-9]?/ GROUP BY time(10s) END`) return s }
// NewService returns an instance of the Graphite service. func NewService(c Config) (*Service, error) { // Use defaults where necessary. d := c.WithDefaults() s := Service{ bindAddress: d.BindAddress, database: d.Database, retentionPolicy: d.RetentionPolicy, protocol: d.Protocol, batchSize: d.BatchSize, batchPending: d.BatchPending, udpReadBuffer: d.UDPReadBuffer, batchTimeout: time.Duration(d.BatchTimeout), logger: zap.New(zap.NullEncoder()), stats: &Statistics{}, defaultTags: models.StatisticTags{"proto": d.Protocol, "bind": d.BindAddress}, tcpConnections: make(map[string]*tcpConnection), diagsKey: strings.Join([]string{"graphite", d.Protocol, d.BindAddress}, ":"), } parser, err := NewParserWithOptions(Options{ Templates: d.Templates, DefaultTags: d.DefaultTags(), Separator: d.Separator}) if err != nil { return nil, err } s.parser = parser return &s, nil }
// NewShard returns a new initialized Shard. walPath doesn't apply to the b1 type index func NewShard(id uint64, index *DatabaseIndex, path string, walPath string, options EngineOptions) *Shard { db, rp := DecodeStorePath(path) logger := zap.New(zap.NullEncoder()) s := &Shard{ index: index, id: id, path: path, walPath: walPath, options: options, closing: make(chan struct{}), stats: &ShardStatistics{}, defaultTags: models.StatisticTags{ "path": path, "walPath": walPath, "id": fmt.Sprintf("%d", id), "database": db, "retentionPolicy": rp, "engine": options.EngineVersion, }, database: db, retentionPolicy: rp, logger: logger, baseLogger: logger, EnableOnOpen: true, } return s }
func NewTestService(batchSize int, batchDuration time.Duration) *TestService { c := Config{ BindAddress: "127.0.0.1:0", Database: "collectd_test", BatchSize: batchSize, BatchDuration: toml.Duration(batchDuration), } s := &TestService{ Config: c, Service: NewService(c), MetaClient: &internal.MetaClientMock{}, } s.MetaClient.CreateDatabaseFn = func(name string) (*meta.DatabaseInfo, error) { return nil, nil } s.Service.PointsWriter = s s.Service.MetaClient = s.MetaClient // Set the collectd types using test string. if err := s.Service.SetTypes(typesDBText); err != nil { panic(err) } if testing.Verbose() { s.Service.WithLogger(zap.New( zap.NewTextEncoder(), zap.Output(os.Stderr), )) } return s }
// NewQueryExecutor returns a new instance of QueryExecutor. func NewQueryExecutor() *QueryExecutor { return &QueryExecutor{ TaskManager: NewTaskManager(), Logger: zap.New(zap.NullEncoder()), stats: &QueryStatistics{}, } }
// NewService returns a configured retention policy enforcement service. func NewService(c Config) *Service { return &Service{ checkInterval: time.Duration(c.CheckInterval), done: make(chan struct{}), logger: zap.New(zap.NullEncoder()), } }
// NewPointsWriter returns a new instance of PointsWriter for a node. func NewPointsWriter() *PointsWriter { return &PointsWriter{ closing: make(chan struct{}), WriteTimeout: DefaultWriteTimeout, Logger: zap.New(zap.NullEncoder()), stats: &WriteStatistics{}, } }
// NewTaskManager creates a new TaskManager. func NewTaskManager() *TaskManager { return &TaskManager{ QueryTimeout: DefaultQueryTimeout, Logger: zap.New(zap.NullEncoder()), queries: make(map[uint64]*QueryTask), nextID: 1, } }
// NewService returns an instance of the precreation service. func NewService(c Config) (*Service, error) { s := Service{ checkInterval: time.Duration(c.CheckInterval), advancePeriod: time.Duration(c.AdvancePeriod), Logger: zap.New(zap.NullEncoder()), } return &s, nil }
// NewService returns a subscriber service with given settings func NewService(c Config) *Service { s := &Service{ Logger: zap.New(zap.NullEncoder()), closed: true, stats: &Statistics{}, conf: c, } s.NewPointsWriter = s.newPointsWriter return s }
// NewService returns a new instance of Service. func NewService(c Config) *Service { return &Service{ addr: c.BindAddress, https: c.HTTPSEnabled, cert: c.HTTPSCertificate, err: make(chan error), version: c.Version, logger: zap.New(zap.NullEncoder()), } }
// NewCommand return a new instance of Command. func NewCommand() *Command { return &Command{ closing: make(chan struct{}), Closed: make(chan struct{}), Stdin: os.Stdin, Stdout: os.Stdout, Stderr: os.Stderr, Logger: zap.New(zap.NullEncoder()), } }
// NewStore returns a new store with the given path and a default configuration. // The returned store must be initialized by calling Open before using it. func NewStore(path string) *Store { opts := NewEngineOptions() logger := zap.New(zap.NullEncoder()) return &Store{ path: path, EngineOptions: opts, Logger: logger, baseLogger: logger, } }
// NewService returns a new instance of Service. func NewService(c Config) *Service { d := *c.WithDefaults() return &Service{ config: d, parserChan: make(chan []byte, parserChanLen), batcher: tsdb.NewPointBatcher(d.BatchSize, d.BatchPending, time.Duration(d.BatchTimeout)), Logger: zap.New(zap.NullEncoder()), stats: &Statistics{}, defaultTags: models.StatisticTags{"bind": d.BindAddress}, } }
// NewMain return a new instance of Main. func NewMain() *Main { return &Main{ Logger: zap.New( zap.NewTextEncoder(), zap.Output(os.Stderr), ), Stdin: os.Stdin, Stdout: os.Stdout, Stderr: os.Stderr, } }
// NewService returns a new instance of the collectd service. func NewService(c Config) *Service { s := Service{ // Use defaults where necessary. Config: c.WithDefaults(), Logger: zap.New(zap.NullEncoder()), stats: &Statistics{}, defaultTags: models.StatisticTags{"bind": c.BindAddress}, } return &s }
// New returns a new instance of the monitor system. func New(r Reporter, c Config) *Monitor { return &Monitor{ globalTags: make(map[string]string), diagRegistrations: make(map[string]diagnostics.Client), reporter: r, storeEnabled: c.StoreEnabled, storeDatabase: c.StoreDatabase, storeInterval: time.Duration(c.StoreInterval), storeRetentionPolicy: MonitorRetentionPolicy, Logger: zap.New(zap.NullEncoder()), } }
// NewService returns a new instance of Service. func NewService(c Config) *Service { s := &Service{ Config: &c, RunInterval: time.Duration(c.RunInterval), RunCh: make(chan *RunRequest), loggingEnabled: c.LogEnabled, Logger: zap.New(zap.NullEncoder()), stats: &Statistics{}, lastRuns: map[string]time.Time{}, } return s }
func NewWAL(path string) *WAL { logger := zap.New(zap.NullEncoder()) return &WAL{ path: path, // these options should be overriden by any options in the config SegmentSize: DefaultSegmentSize, closing: make(chan struct{}), stats: &WALStatistics{}, limiter: limiter.NewFixed(defaultWaitingWALWrites), logger: logger, traceLogger: logger, } }
// NewClient returns a new *Client. func NewClient(config *Config) *Client { return &Client{ cacheData: &Data{ ClusterID: uint64(rand.Int63()), Index: 1, }, closing: make(chan struct{}), changed: make(chan struct{}), logger: zap.New(zap.NullEncoder()), authCache: make(map[string]authUser, 0), path: config.Dir, retentionAutoCreate: config.RetentionAutoCreate, } }
// NewHandler returns a new instance of handler with routes. func NewHandler(c Config) *Handler { h := &Handler{ mux: pat.New(), Config: &c, Logger: zap.New(zap.NullEncoder()), CLFLogger: log.New(os.Stderr, "[httpd] ", 0), stats: &Statistics{}, } h.AddRoutes([]Route{ Route{ "query-options", // Satisfy CORS checks. "OPTIONS", "/query", false, true, h.serveOptions, }, Route{ "query", // Query serving route. "GET", "/query", true, true, h.serveQuery, }, Route{ "query", // Query serving route. "POST", "/query", true, true, h.serveQuery, }, Route{ "write-options", // Satisfy CORS checks. "OPTIONS", "/write", false, true, h.serveOptions, }, Route{ "write", // Data-ingest route. "POST", "/write", true, true, h.serveWrite, }, Route{ // Ping "ping", "GET", "/ping", false, true, h.servePing, }, Route{ // Ping "ping-head", "HEAD", "/ping", false, true, h.servePing, }, Route{ // Ping w/ status "status", "GET", "/status", false, true, h.serveStatus, }, Route{ // Ping w/ status "status-head", "HEAD", "/status", false, true, h.serveStatus, }, }...) return h }
func NewFileStore(dir string) *FileStore { logger := zap.New(zap.NullEncoder()) fs := &FileStore{ dir: dir, lastModified: time.Time{}, logger: logger, traceLogger: logger, stats: &FileStoreStatistics{}, purger: &purger{ files: map[string]TSMFile{}, logger: logger, }, } fs.purger.fileStore = fs return fs }
func NewTestService(c *Config) *TestService { if c == nil { defaultC := NewConfig() c = &defaultC } gservice, err := NewService(*c) if err != nil { panic(err) } service := &TestService{ Service: gservice, MetaClient: &internal.MetaClientMock{}, } service.MetaClient.CreateRetentionPolicyFn = func(string, *meta.RetentionPolicySpec, bool) (*meta.RetentionPolicyInfo, error) { return nil, nil } service.MetaClient.CreateDatabaseWithRetentionPolicyFn = func(string, *meta.RetentionPolicySpec) (*meta.DatabaseInfo, error) { return nil, nil } service.MetaClient.DatabaseFn = func(string) *meta.DatabaseInfo { return nil } service.MetaClient.RetentionPolicyFn = func(string, string) (*meta.RetentionPolicyInfo, error) { return nil, nil } if testing.Verbose() { service.Service.WithLogger(zap.New( zap.NewTextEncoder(), zap.Output(os.Stderr), )) } // Set the Meta Client and PointsWriter. service.Service.MetaClient = service.MetaClient service.Service.PointsWriter = service return service }
// NewEngine returns a new instance of Engine. func NewEngine(id uint64, path string, walPath string, opt tsdb.EngineOptions) tsdb.Engine { w := NewWAL(walPath) fs := NewFileStore(path) cache := NewCache(uint64(opt.Config.CacheMaxMemorySize), path) c := &Compactor{ Dir: path, FileStore: fs, } logger := zap.New(zap.NullEncoder()) e := &Engine{ id: id, path: path, logger: logger, traceLogger: logger, traceLogging: opt.Config.TraceLoggingEnabled, measurementFields: make(map[string]*tsdb.MeasurementFields), WAL: w, Cache: cache, FileStore: fs, Compactor: c, CompactionPlan: &DefaultPlanner{ FileStore: fs, CompactFullWriteColdDuration: time.Duration(opt.Config.CompactFullWriteColdDuration), }, CacheFlushMemorySizeThreshold: opt.Config.CacheSnapshotMemorySize, CacheFlushWriteColdDuration: time.Duration(opt.Config.CacheSnapshotWriteColdDuration), enableCompactionsOnOpen: true, stats: &EngineStatistics{}, } if e.traceLogging { fs.enableTraceLogging(true) w.enableTraceLogging(true) } return e }
// NewService returns a new instance of Service. func NewService(c Config) *Service { s := &Service{ addr: c.BindAddress, https: c.HTTPSEnabled, cert: c.HTTPSCertificate, key: c.HTTPSPrivateKey, limit: c.MaxConnectionLimit, err: make(chan error), unixSocket: c.UnixSocketEnabled, bindSocket: c.BindSocket, Handler: NewHandler(c), Logger: zap.New(zap.NullEncoder()), } if s.key == "" { s.key = s.cert } s.Handler.Logger = s.Logger return s }
// NewService returns a new instance of Service. func NewService(c Config) (*Service, error) { // Use defaults where necessary. d := c.WithDefaults() s := &Service{ tls: d.TLSEnabled, cert: d.Certificate, BindAddress: d.BindAddress, Database: d.Database, RetentionPolicy: d.RetentionPolicy, batchSize: d.BatchSize, batchPending: d.BatchPending, batchTimeout: time.Duration(d.BatchTimeout), Logger: zap.New(zap.NullEncoder()), LogPointErrors: d.LogPointErrors, stats: &Statistics{}, defaultTags: models.StatisticTags{"bind": d.BindAddress}, } return s, nil }
// NewQueryExecutor returns a new instance of QueryExecutor. // This query executor always has a node id of 0. func NewQueryExecutor() *QueryExecutor { e := &QueryExecutor{ QueryExecutor: influxql.NewQueryExecutor(), } e.StatementExecutor = &coordinator.StatementExecutor{ MetaClient: &e.MetaClient, TSDBStore: &e.TSDBStore, } e.QueryExecutor.StatementExecutor = e.StatementExecutor var out io.Writer = &e.LogOutput if testing.Verbose() { out = io.MultiWriter(out, os.Stderr) } e.QueryExecutor.WithLogger(zap.New( zap.NewTextEncoder(), zap.Output(os.Stderr), )) return e }
func TestWriteTimeField(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "shard_test") defer os.RemoveAll(tmpDir) tmpShard := path.Join(tmpDir, "shard") tmpWal := path.Join(tmpDir, "wal") index := tsdb.NewDatabaseIndex("db") opts := tsdb.NewEngineOptions() opts.Config.WALDir = filepath.Join(tmpDir, "wal") sh := tsdb.NewShard(1, index, tmpShard, tmpWal, opts) if err := sh.Open(); err != nil { t.Fatalf("error opening shard: %s", err.Error()) } defer sh.Close() pt := models.MustNewPoint( "cpu", models.NewTags(map[string]string{"time": "now"}), map[string]interface{}{"value": 1.0}, time.Unix(1, 2), ) buf := bytes.NewBuffer(nil) sh.WithLogger(zap.New(zap.NewTextEncoder(), zap.Output(zap.AddSync(buf)))) if err := sh.WritePoints([]models.Point{pt}); err != nil { t.Fatalf("unexpected error: %v", err) } else if got, exp := buf.String(), "dropping tag 'time'"; !strings.Contains(got, exp) { t.Fatalf("unexpected log message: %s", strings.TrimSpace(got)) } key := models.MakeKey([]byte("cpu"), nil) series := index.Series(string(key)) if series == nil { t.Fatal("expected series") } else if len(series.Tags) != 0 { t.Fatalf("unexpected number of tags: got=%v exp=%v", len(series.Tags), 0) } }