// Test that a registered stats client results in the correct SHOW STATS output.
func Test_RegisterStats(t *testing.T) {
monitor := openMonitor(t)
executor := &StatementExecutor{Monitor: monitor}
// Register stats without tags.
statMap := influxdb.NewStatistics("foo", "foo", nil)
statMap.Add("bar", 1)
statMap.AddFloat("qux", 2.4)
json := executeShowStatsJSON(t, executor)
if !strings.Contains(json, `"columns":["bar","qux"],"values":[[1,2.4]]`) || !strings.Contains(json, `"name":"foo"`) {
t.Fatalf("SHOW STATS response incorrect, got: %s\n", json)
}
// Register a client with tags.
statMap = influxdb.NewStatistics("bar", "baz", map[string]string{"proto": "tcp"})
statMap.Add("bar", 1)
statMap.AddFloat("qux", 2.4)
json = executeShowStatsJSON(t, executor)
if !strings.Contains(json, `"columns":["bar","qux"],"values":[[1,2.4]]`) ||
!strings.Contains(json, `"name":"baz"`) ||
!strings.Contains(json, `"proto":"tcp"`) {
t.Fatalf("SHOW STATS response incorrect, got: %s\n", json)
}
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
return &Service{
closing: make(chan struct{}),
Logger: log.New(os.Stderr, "[cluster] ", log.LstdFlags),
statMap: influxdb.NewStatistics("cluster", "cluster", nil),
}
}
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
return &QueryExecutor{
TaskManager: NewTaskManager(),
Logger: log.New(ioutil.Discard, "[query] ", log.LstdFlags),
statMap: influxdb.NewStatistics("queryExecutor", "queryExecutor", nil),
}
}
// NewShard returns a new initialized Shard. walPath doesn't apply to the b1 type index
func NewShard(id uint64, index *DatabaseIndex, path string, walPath string, options EngineOptions) *Shard {
// Configure statistics collection.
key := fmt.Sprintf("shard:%s:%d", path, id)
db, rp := DecodeStorePath(path)
tags := map[string]string{
"path": path,
"id": fmt.Sprintf("%d", id),
"engine": options.EngineVersion,
"database": db,
"retentionPolicy": rp,
}
statMap := influxdb.NewStatistics(key, "shard", tags)
return &Shard{
index: index,
id: id,
path: path,
walPath: walPath,
options: options,
database: db,
retentionPolicy: rp,
statMap: statMap,
LogOutput: os.Stderr,
}
}
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
return &QueryExecutor{
Timeout: DefaultShardMapperTimeout,
LogOutput: ioutil.Discard,
statMap: influxdb.NewStatistics("queryExecutor", "queryExecutor", nil),
}
}
// NewDatabaseIndex returns a new initialized DatabaseIndex.
func NewDatabaseIndex(name string) *DatabaseIndex {
return &DatabaseIndex{
measurements: make(map[string]*Measurement),
series: make(map[string]*Series),
name: name,
statMap: influxdb.NewStatistics("database:"+name, "database", map[string]string{"database": name}),
}
}
// NewPointsWriter returns a new instance of PointsWriter for a node.
func NewPointsWriter() *PointsWriter {
return &PointsWriter{
closing: make(chan struct{}),
WriteTimeout: DefaultWriteTimeout,
Logger: log.New(os.Stderr, "[write] ", log.LstdFlags),
statMap: influxdb.NewStatistics("write", "write", nil),
}
}
// Open starts the service
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
s.Logger.Println("Starting OpenTSDB service")
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
key := strings.Join([]string{"opentsdb", s.BindAddress}, ":")
tags := map[string]string{"bind": s.BindAddress}
s.statMap = influxdb.NewStatistics(key, "opentsdb", tags)
if _, err := s.MetaClient.CreateDatabase(s.Database); err != nil {
s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Database, err.Error())
return err
}
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go s.processBatches(s.batcher)
// Open listener.
if s.tls {
cert, err := tls.LoadX509KeyPair(s.cert, s.cert)
if err != nil {
return err
}
listener, err := tls.Listen("tcp", s.BindAddress, &tls.Config{
Certificates: []tls.Certificate{cert},
})
if err != nil {
return err
}
s.Logger.Println("Listening on TLS:", listener.Addr().String())
s.ln = listener
} else {
listener, err := net.Listen("tcp", s.BindAddress)
if err != nil {
return err
}
s.Logger.Println("Listening on:", listener.Addr().String())
s.ln = listener
}
s.httpln = newChanListener(s.ln.Addr())
// Begin listening for connections.
s.wg.Add(2)
go s.serveHTTP()
go s.serve()
return nil
}
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
return &QueryExecutor{
QueryTimeout: DefaultQueryTimeout,
Logger: log.New(ioutil.Discard, "[query] ", log.LstdFlags),
queries: make(map[uint64]*QueryTask),
nextID: 1,
statMap: influxdb.NewStatistics("queryExecutor", "queryExecutor", nil),
}
}
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
return &QueryExecutor{
QueryTimeout: DefaultQueryTimeout,
LogOutput: ioutil.Discard,
queries: make(map[uint64]*QueryTask),
nextID: 1,
statMap: influxdb.NewStatistics("queryExecutor", "queryExecutor", nil),
}
}
// NewHandler returns a new instance of Handler.
func NewHandler(requireAuthentication bool) *Handler {
statMap := influxdb.NewStatistics("httpd", "httpd", nil)
h := &Handler{
Handler: httpd.NewHandler(requireAuthentication, true, false, statMap),
}
h.Handler.MetaClient = &h.MetaClient
h.Handler.QueryExecutor = &h.QueryExecutor
h.Handler.Version = "0.0.0"
return h
}
// NewService returns a subscriber service with given settings
func NewService(c Config) *Service {
return &Service{
subs: make(map[subEntry]PointsWriter),
NewPointsWriter: newPointsWriter,
Logger: log.New(os.Stderr, "[subscriber] ", log.LstdFlags),
statMap: influxdb.NewStatistics("subscriber", "subscriber", nil),
points: make(chan *cluster.WritePointsRequest),
closed: true,
closing: make(chan struct{}),
}
}
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
tags := map[string]string{"proto": s.protocol, "bind": s.bindAddress}
s.statMap = influxdb.NewStatistics(s.diagsKey, "graphite", tags)
// Register diagnostics if a Monitor service is available.
if s.Monitor != nil {
s.Monitor.RegisterDiagnosticsClient(s.diagsKey, s)
}
if db := s.MetaClient.Database(s.database); db != nil {
if rp, _ := s.MetaClient.RetentionPolicy(s.database, s.retentionPolicy); rp == nil {
rpi := meta.NewRetentionPolicyInfo(s.retentionPolicy)
if _, err := s.MetaClient.CreateRetentionPolicy(s.database, rpi); err != nil {
s.logger.Printf("Failed to ensure target retention policy %s exists: %s", s.database, err.Error())
}
}
} else {
rpi := meta.NewRetentionPolicyInfo(s.retentionPolicy)
if _, err := s.MetaClient.CreateDatabaseWithRetentionPolicy(s.database, rpi); err != nil {
s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
return err
}
}
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go s.processBatches(s.batcher)
var err error
if strings.ToLower(s.protocol) == "tcp" {
s.addr, err = s.openTCPServer()
} else if strings.ToLower(s.protocol) == "udp" {
s.addr, err = s.openUDPServer()
} else {
return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
}
if err != nil {
return err
}
s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
return nil
}
func NewFileStore(dir string) *FileStore {
db, rp := tsdb.DecodeStorePath(dir)
return &FileStore{
dir: dir,
lastModified: time.Now(),
Logger: log.New(os.Stderr, "[filestore] ", log.LstdFlags),
statMap: influxdb.NewStatistics(
"tsm1_filestore:"+dir,
"tsm1_filestore",
map[string]string{"path": dir, "database": db, "retentionPolicy": rp},
),
}
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
s := &Service{
Config: &c,
RunInterval: time.Duration(c.RunInterval),
RunCh: make(chan *RunRequest),
loggingEnabled: c.LogEnabled,
statMap: influxdb.NewStatistics("cq", "cq", nil),
Logger: log.New(os.Stderr, "[continuous_querier] ", log.LstdFlags),
lastRuns: map[string]time.Time{},
}
return s
}
// NewService returns a new instance of Service.
func NewService(c Config, w shardWriter, m metaClient) *Service {
key := strings.Join([]string{"hh", c.Dir}, ":")
tags := map[string]string{"path": c.Dir}
return &Service{
cfg: c,
closing: make(chan struct{}),
processors: make(map[uint64]*NodeProcessor),
statMap: influxdb.NewStatistics(key, "hh", tags),
Logger: log.New(os.Stderr, "[handoff] ", log.LstdFlags),
shardWriter: w,
MetaClient: m,
}
}
// NewService returns a subscriber service with given settings
func NewService(c Config) *Service {
s := &Service{
Logger: log.New(os.Stderr, "[subscriber] ", log.LstdFlags),
statMap: influxdb.NewStatistics("subscriber", "subscriber", nil),
closed: true,
conf: c,
failures: &expvar.Int{},
pointsWritten: &expvar.Int{},
}
s.NewPointsWriter = s.newPointsWriter
s.statMap.Set(statWriteFailures, s.failures)
s.statMap.Set(statPointsWritten, s.pointsWritten)
return s
}
// Open starts the service
func (s *Service) Open() (err error) {
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
key := strings.Join([]string{"udp", s.config.BindAddress}, ":")
tags := map[string]string{"bind": s.config.BindAddress}
s.statMap = influxdb.NewStatistics(key, "udp", tags)
if s.config.BindAddress == "" {
return errors.New("bind address has to be specified in config")
}
if s.config.Database == "" {
return errors.New("database has to be specified in config")
}
if _, err := s.MetaClient.CreateDatabase(s.config.Database); err != nil {
return errors.New("Failed to ensure target database exists")
}
s.addr, err = net.ResolveUDPAddr("udp", s.config.BindAddress)
if err != nil {
s.Logger.Printf("Failed to resolve UDP address %s: %s", s.config.BindAddress, err)
return err
}
s.conn, err = net.ListenUDP("udp", s.addr)
if err != nil {
s.Logger.Printf("Failed to set up UDP listener at address %s: %s", s.addr, err)
return err
}
if s.config.ReadBuffer != 0 {
err = s.conn.SetReadBuffer(s.config.ReadBuffer)
if err != nil {
s.Logger.Printf("Failed to set UDP read buffer to %d: %s",
s.config.ReadBuffer, err)
return err
}
}
s.Logger.Printf("Started listening on UDP: %s", s.config.BindAddress)
s.wg.Add(3)
go s.serve()
go s.parser()
go s.writer()
return nil
}
// NewHandler returns a new instance of Handler.
func NewHandler(requireAuthentication bool) *Handler {
statMap := influxdb.NewStatistics("httpd", "httpd", nil)
config := httpd.NewConfig()
config.AuthEnabled = requireAuthentication
config.SharedSecret = "super secret key"
h := &Handler{
Handler: httpd.NewHandler(config, statMap),
}
h.Handler.MetaClient = &h.MetaClient
h.Handler.QueryExecutor = influxql.NewQueryExecutor()
h.Handler.QueryExecutor.StatementExecutor = &h.StatementExecutor
h.Handler.QueryAuthorizer = &h.QueryAuthorizer
h.Handler.Version = "0.0.0"
return h
}
// NewMeasurement allocates and initializes a new Measurement.
func NewMeasurement(name string, idx *DatabaseIndex) *Measurement {
return &Measurement{
Name: name,
fieldNames: make(map[string]struct{}),
index: idx,
seriesByID: make(map[uint64]*Series),
seriesByTagKeyValue: make(map[string]map[string]SeriesIDs),
seriesIDs: make(SeriesIDs, 0),
statMap: influxdb.NewStatistics(
fmt.Sprintf("measurement:%s.%s", name, idx.name),
"measurement",
map[string]string{"database": idx.name, "measurement": name},
),
}
}
// NewShard returns a new initialized Shard. walPath doesn't apply to the b1 type index
func NewShard(id uint64, index *DatabaseIndex, path string, walPath string, options EngineOptions) *Shard {
// Configure statistics collection.
key := fmt.Sprintf("shard:%s:%d", path, id)
tags := map[string]string{"path": path, "id": fmt.Sprintf("%d", id), "engine": options.EngineVersion}
statMap := influxdb.NewStatistics(key, "shard", tags)
return &Shard{
index: index,
path: path,
walPath: walPath,
id: id,
options: options,
measurementFields: make(map[string]*MeasurementFields),
statMap: statMap,
LogOutput: os.Stderr,
}
}
// NewNodeProcessor returns a new NodeProcessor for the given node, using dir for
// the hinted-handoff data.
func NewNodeProcessor(nodeID uint64, dir string, w shardWriter, m metaClient) *NodeProcessor {
key := strings.Join([]string{"hh_processor", dir}, ":")
tags := map[string]string{"node": fmt.Sprintf("%d", nodeID), "path": dir}
return &NodeProcessor{
PurgeInterval: DefaultPurgeInterval,
RetryInterval: DefaultRetryInterval,
RetryMaxInterval: DefaultRetryMaxInterval,
MaxSize: DefaultMaxSize,
MaxAge: DefaultMaxAge,
nodeID: nodeID,
dir: dir,
writer: w,
meta: m,
statMap: influxdb.NewStatistics(key, "hh_processor", tags),
Logger: log.New(os.Stderr, "[handoff] ", log.LstdFlags),
}
}
func NewWAL(path string) *WAL {
db, rp := tsdb.DecodeStorePath(path)
return &WAL{
path: path,
// these options should be overriden by any options in the config
LogOutput: os.Stderr,
SegmentSize: DefaultSegmentSize,
logger: log.New(os.Stderr, "[tsm1wal] ", log.LstdFlags),
closing: make(chan struct{}),
statMap: influxdb.NewStatistics(
"tsm1_wal:"+path,
"tsm1_wal",
map[string]string{"path": path, "database": db, "retentionPolicy": rp},
),
}
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
key := strings.Join([]string{"httpd", c.BindAddress}, ":")
tags := map[string]string{"bind": c.BindAddress}
statMap := influxdb.NewStatistics(key, "httpd", tags)
s := &Service{
addr: c.BindAddress,
https: c.HTTPSEnabled,
cert: c.HTTPSCertificate,
limit: c.MaxConnectionLimit,
err: make(chan error),
Handler: NewHandler(c, statMap),
Logger: log.New(os.Stderr, "[httpd] ", log.LstdFlags),
}
s.Handler.Logger = s.Logger
return s
}
// NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory.
// Only used for engine caches, never for snapshots
func NewCache(maxSize uint64, path string) *Cache {
db, rp := tsdb.DecodeStorePath(path)
c := &Cache{
maxSize: maxSize,
store: make(map[string]*entry),
statMap: influxdb.NewStatistics(
"tsm1_cache:"+path,
"tsm1_cache",
map[string]string{"path": path, "database": db, "retentionPolicy": rp},
),
lastSnapshot: time.Now(),
}
c.UpdateAge()
c.UpdateCompactTime(0)
c.updateCachedBytes(0)
c.updateMemSize(0)
c.updateSnapshots()
return c
}
func (s *Service) createSubscription(se subEntry, mode string, destinations []string) (PointsWriter, error) {
var bm BalanceMode
switch mode {
case "ALL":
bm = ALL
case "ANY":
bm = ANY
default:
return nil, fmt.Errorf("unknown balance mode %q", mode)
}
writers := make([]PointsWriter, len(destinations))
statMaps := make([]*expvar.Map, len(writers))
for i, dest := range destinations {
u, err := url.Parse(dest)
if err != nil {
return nil, err
}
w, err := s.NewPointsWriter(*u)
if err != nil {
return nil, err
}
writers[i] = w
tags := map[string]string{
"database": se.db,
"retention_policy": se.rp,
"name": se.name,
"mode": mode,
"destination": dest,
}
key := strings.Join([]string{"subscriber", se.db, se.rp, se.name, dest}, ":")
statMaps[i] = influxdb.NewStatistics(key, "subscriber", tags)
}
s.Logger.Println("created new subscription for", se.db, se.rp)
return &balancewriter{
bm: bm,
writers: writers,
statMaps: statMaps,
}, nil
}
// NewLog creates a new Log object with certain default values.
func NewLog(path string) *Log {
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
key := strings.Join([]string{"wal", path}, ":")
tags := map[string]string{"path": path}
return &Log{
path: path,
flush: make(chan int, 1),
// these options should be overriden by any options in the config
LogOutput: os.Stderr,
FlushColdInterval: tsdb.DefaultFlushColdInterval,
SegmentSize: DefaultSegmentSize,
MaxSeriesSize: tsdb.DefaultMaxSeriesSize,
CompactionThreshold: tsdb.DefaultCompactionThreshold,
PartitionSizeThreshold: tsdb.DefaultPartitionSizeThreshold,
ReadySeriesSize: tsdb.DefaultReadySeriesSize,
flushCheckInterval: defaultFlushCheckInterval,
logger: log.New(os.Stderr, "[wal] ", log.LstdFlags),
statMap: influxdb.NewStatistics(key, "wal", tags),
}
}
// Open starts the service.
func (s *Service) Open() error {
s.Logger.Printf("Starting collectd service")
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
key := strings.Join([]string{"collectd", s.Config.BindAddress}, ":")
tags := map[string]string{"bind": s.Config.BindAddress}
s.statMap = influxdb.NewStatistics(key, "collectd", tags)
if s.Config.BindAddress == "" {
return fmt.Errorf("bind address is blank")
} else if s.Config.Database == "" {
return fmt.Errorf("database name is blank")
} else if s.PointsWriter == nil {
return fmt.Errorf("PointsWriter is nil")
}
if _, err := s.MetaClient.CreateDatabase(s.Config.Database); err != nil {
s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Config.Database, err.Error())
return err
}
if s.typesdb == nil {
// Open collectd types.
typesdb, err := gollectd.TypesDBFile(s.Config.TypesDB)
if err != nil {
return fmt.Errorf("Open(): %s", err)
}
s.typesdb = typesdb
}
// Resolve our address.
addr, err := net.ResolveUDPAddr("udp", s.Config.BindAddress)
if err != nil {
return fmt.Errorf("unable to resolve UDP address: %s", err)
}
s.addr = addr
// Start listening
conn, err := net.ListenUDP("udp", addr)
if err != nil {
return fmt.Errorf("unable to listen on UDP: %s", err)
}
if s.Config.ReadBuffer != 0 {
err = conn.SetReadBuffer(s.Config.ReadBuffer)
if err != nil {
return fmt.Errorf("unable to set UDP read buffer to %d: %s",
s.Config.ReadBuffer, err)
}
}
s.conn = conn
s.Logger.Println("Listening on UDP: ", conn.LocalAddr().String())
// Start the points batcher.
s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, s.Config.BatchPending, time.Duration(s.Config.BatchDuration))
s.batcher.Start()
// Create channel and wait group for signalling goroutines to stop.
s.stop = make(chan struct{})
s.wg.Add(2)
// Start goroutines that process collectd packets.
go s.serve()
go s.writePoints()
return nil
}