Exemplo n.º 1
0
// Test that a registered stats client results in the correct SHOW STATS output.
func Test_RegisterStats(t *testing.T) {
	monitor := openMonitor(t)
	executor := &StatementExecutor{Monitor: monitor}

	// Register stats without tags.
	statMap := influxdb.NewStatistics("foo", "foo", nil)
	statMap.Add("bar", 1)
	statMap.AddFloat("qux", 2.4)
	json := executeShowStatsJSON(t, executor)
	if !strings.Contains(json, `"columns":["bar","qux"],"values":[[1,2.4]]`) || !strings.Contains(json, `"name":"foo"`) {
		t.Fatalf("SHOW STATS response incorrect, got: %s\n", json)
	}

	// Register a client with tags.
	statMap = influxdb.NewStatistics("bar", "baz", map[string]string{"proto": "tcp"})
	statMap.Add("bar", 1)
	statMap.AddFloat("qux", 2.4)
	json = executeShowStatsJSON(t, executor)
	if !strings.Contains(json, `"columns":["bar","qux"],"values":[[1,2.4]]`) ||
		!strings.Contains(json, `"name":"baz"`) ||
		!strings.Contains(json, `"proto":"tcp"`) {
		t.Fatalf("SHOW STATS response incorrect, got: %s\n", json)

	}
}
Exemplo n.º 2
0
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
	return &Service{
		closing: make(chan struct{}),
		Logger:  log.New(os.Stderr, "[cluster] ", log.LstdFlags),
		statMap: influxdb.NewStatistics("cluster", "cluster", nil),
	}
}
Exemplo n.º 3
0
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
	return &QueryExecutor{
		TaskManager: NewTaskManager(),
		Logger:      log.New(ioutil.Discard, "[query] ", log.LstdFlags),
		statMap:     influxdb.NewStatistics("queryExecutor", "queryExecutor", nil),
	}
}
Exemplo n.º 4
0
// NewShard returns a new initialized Shard. walPath doesn't apply to the b1 type index
func NewShard(id uint64, index *DatabaseIndex, path string, walPath string, options EngineOptions) *Shard {
	// Configure statistics collection.
	key := fmt.Sprintf("shard:%s:%d", path, id)
	db, rp := DecodeStorePath(path)
	tags := map[string]string{
		"path":            path,
		"id":              fmt.Sprintf("%d", id),
		"engine":          options.EngineVersion,
		"database":        db,
		"retentionPolicy": rp,
	}
	statMap := influxdb.NewStatistics(key, "shard", tags)

	return &Shard{
		index:   index,
		id:      id,
		path:    path,
		walPath: walPath,
		options: options,

		database:        db,
		retentionPolicy: rp,

		statMap:   statMap,
		LogOutput: os.Stderr,
	}
}
Exemplo n.º 5
0
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
	return &QueryExecutor{
		Timeout:   DefaultShardMapperTimeout,
		LogOutput: ioutil.Discard,
		statMap:   influxdb.NewStatistics("queryExecutor", "queryExecutor", nil),
	}
}
Exemplo n.º 6
0
// NewDatabaseIndex returns a new initialized DatabaseIndex.
func NewDatabaseIndex(name string) *DatabaseIndex {
	return &DatabaseIndex{
		measurements: make(map[string]*Measurement),
		series:       make(map[string]*Series),
		name:         name,
		statMap:      influxdb.NewStatistics("database:"+name, "database", map[string]string{"database": name}),
	}
}
Exemplo n.º 7
0
// NewPointsWriter returns a new instance of PointsWriter for a node.
func NewPointsWriter() *PointsWriter {
	return &PointsWriter{
		closing:      make(chan struct{}),
		WriteTimeout: DefaultWriteTimeout,
		Logger:       log.New(os.Stderr, "[write] ", log.LstdFlags),
		statMap:      influxdb.NewStatistics("write", "write", nil),
	}
}
Exemplo n.º 8
0
// Open starts the service
func (s *Service) Open() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.Logger.Println("Starting OpenTSDB service")

	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	key := strings.Join([]string{"opentsdb", s.BindAddress}, ":")
	tags := map[string]string{"bind": s.BindAddress}
	s.statMap = influxdb.NewStatistics(key, "opentsdb", tags)

	if _, err := s.MetaClient.CreateDatabase(s.Database); err != nil {
		s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Database, err.Error())
		return err
	}

	s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
	s.batcher.Start()

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(s.batcher)

	// Open listener.
	if s.tls {
		cert, err := tls.LoadX509KeyPair(s.cert, s.cert)
		if err != nil {
			return err
		}

		listener, err := tls.Listen("tcp", s.BindAddress, &tls.Config{
			Certificates: []tls.Certificate{cert},
		})
		if err != nil {
			return err
		}

		s.Logger.Println("Listening on TLS:", listener.Addr().String())
		s.ln = listener
	} else {
		listener, err := net.Listen("tcp", s.BindAddress)
		if err != nil {
			return err
		}

		s.Logger.Println("Listening on:", listener.Addr().String())
		s.ln = listener
	}
	s.httpln = newChanListener(s.ln.Addr())

	// Begin listening for connections.
	s.wg.Add(2)
	go s.serveHTTP()
	go s.serve()

	return nil
}
Exemplo n.º 9
0
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
	return &QueryExecutor{
		QueryTimeout: DefaultQueryTimeout,
		Logger:       log.New(ioutil.Discard, "[query] ", log.LstdFlags),
		queries:      make(map[uint64]*QueryTask),
		nextID:       1,
		statMap:      influxdb.NewStatistics("queryExecutor", "queryExecutor", nil),
	}
}
Exemplo n.º 10
0
// NewQueryExecutor returns a new instance of QueryExecutor.
func NewQueryExecutor() *QueryExecutor {
	return &QueryExecutor{
		QueryTimeout: DefaultQueryTimeout,
		LogOutput:    ioutil.Discard,
		queries:      make(map[uint64]*QueryTask),
		nextID:       1,
		statMap:      influxdb.NewStatistics("queryExecutor", "queryExecutor", nil),
	}
}
Exemplo n.º 11
0
// NewHandler returns a new instance of Handler.
func NewHandler(requireAuthentication bool) *Handler {
	statMap := influxdb.NewStatistics("httpd", "httpd", nil)
	h := &Handler{
		Handler: httpd.NewHandler(requireAuthentication, true, false, statMap),
	}
	h.Handler.MetaClient = &h.MetaClient
	h.Handler.QueryExecutor = &h.QueryExecutor
	h.Handler.Version = "0.0.0"
	return h
}
Exemplo n.º 12
0
// NewService returns a subscriber service with given settings
func NewService(c Config) *Service {
	return &Service{
		subs:            make(map[subEntry]PointsWriter),
		NewPointsWriter: newPointsWriter,
		Logger:          log.New(os.Stderr, "[subscriber] ", log.LstdFlags),
		statMap:         influxdb.NewStatistics("subscriber", "subscriber", nil),
		points:          make(chan *cluster.WritePointsRequest),
		closed:          true,
		closing:         make(chan struct{}),
	}
}
Exemplo n.º 13
0
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)

	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	tags := map[string]string{"proto": s.protocol, "bind": s.bindAddress}
	s.statMap = influxdb.NewStatistics(s.diagsKey, "graphite", tags)

	// Register diagnostics if a Monitor service is available.
	if s.Monitor != nil {
		s.Monitor.RegisterDiagnosticsClient(s.diagsKey, s)
	}

	if db := s.MetaClient.Database(s.database); db != nil {
		if rp, _ := s.MetaClient.RetentionPolicy(s.database, s.retentionPolicy); rp == nil {
			rpi := meta.NewRetentionPolicyInfo(s.retentionPolicy)
			if _, err := s.MetaClient.CreateRetentionPolicy(s.database, rpi); err != nil {
				s.logger.Printf("Failed to ensure target retention policy %s exists: %s", s.database, err.Error())
			}
		}
	} else {
		rpi := meta.NewRetentionPolicyInfo(s.retentionPolicy)
		if _, err := s.MetaClient.CreateDatabaseWithRetentionPolicy(s.database, rpi); err != nil {
			s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
			return err
		}
	}

	s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
	s.batcher.Start()

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(s.batcher)

	var err error
	if strings.ToLower(s.protocol) == "tcp" {
		s.addr, err = s.openTCPServer()
	} else if strings.ToLower(s.protocol) == "udp" {
		s.addr, err = s.openUDPServer()
	} else {
		return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
	}
	if err != nil {
		return err
	}

	s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
	return nil
}
Exemplo n.º 14
0
func NewFileStore(dir string) *FileStore {
	db, rp := tsdb.DecodeStorePath(dir)
	return &FileStore{
		dir:          dir,
		lastModified: time.Now(),
		Logger:       log.New(os.Stderr, "[filestore] ", log.LstdFlags),
		statMap: influxdb.NewStatistics(
			"tsm1_filestore:"+dir,
			"tsm1_filestore",
			map[string]string{"path": dir, "database": db, "retentionPolicy": rp},
		),
	}
}
Exemplo n.º 15
0
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
	s := &Service{
		Config:         &c,
		RunInterval:    time.Duration(c.RunInterval),
		RunCh:          make(chan *RunRequest),
		loggingEnabled: c.LogEnabled,
		statMap:        influxdb.NewStatistics("cq", "cq", nil),
		Logger:         log.New(os.Stderr, "[continuous_querier] ", log.LstdFlags),
		lastRuns:       map[string]time.Time{},
	}

	return s
}
Exemplo n.º 16
0
// NewService returns a new instance of Service.
func NewService(c Config, w shardWriter, m metaClient) *Service {
	key := strings.Join([]string{"hh", c.Dir}, ":")
	tags := map[string]string{"path": c.Dir}

	return &Service{
		cfg:         c,
		closing:     make(chan struct{}),
		processors:  make(map[uint64]*NodeProcessor),
		statMap:     influxdb.NewStatistics(key, "hh", tags),
		Logger:      log.New(os.Stderr, "[handoff] ", log.LstdFlags),
		shardWriter: w,
		MetaClient:  m,
	}
}
Exemplo n.º 17
0
// NewService returns a subscriber service with given settings
func NewService(c Config) *Service {
	s := &Service{
		Logger:        log.New(os.Stderr, "[subscriber] ", log.LstdFlags),
		statMap:       influxdb.NewStatistics("subscriber", "subscriber", nil),
		closed:        true,
		conf:          c,
		failures:      &expvar.Int{},
		pointsWritten: &expvar.Int{},
	}
	s.NewPointsWriter = s.newPointsWriter
	s.statMap.Set(statWriteFailures, s.failures)
	s.statMap.Set(statPointsWritten, s.pointsWritten)
	return s
}
Exemplo n.º 18
0
// Open starts the service
func (s *Service) Open() (err error) {
	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	key := strings.Join([]string{"udp", s.config.BindAddress}, ":")
	tags := map[string]string{"bind": s.config.BindAddress}
	s.statMap = influxdb.NewStatistics(key, "udp", tags)

	if s.config.BindAddress == "" {
		return errors.New("bind address has to be specified in config")
	}
	if s.config.Database == "" {
		return errors.New("database has to be specified in config")
	}

	if _, err := s.MetaClient.CreateDatabase(s.config.Database); err != nil {
		return errors.New("Failed to ensure target database exists")
	}

	s.addr, err = net.ResolveUDPAddr("udp", s.config.BindAddress)
	if err != nil {
		s.Logger.Printf("Failed to resolve UDP address %s: %s", s.config.BindAddress, err)
		return err
	}

	s.conn, err = net.ListenUDP("udp", s.addr)
	if err != nil {
		s.Logger.Printf("Failed to set up UDP listener at address %s: %s", s.addr, err)
		return err
	}

	if s.config.ReadBuffer != 0 {
		err = s.conn.SetReadBuffer(s.config.ReadBuffer)
		if err != nil {
			s.Logger.Printf("Failed to set UDP read buffer to %d: %s",
				s.config.ReadBuffer, err)
			return err
		}
	}

	s.Logger.Printf("Started listening on UDP: %s", s.config.BindAddress)

	s.wg.Add(3)
	go s.serve()
	go s.parser()
	go s.writer()

	return nil
}
Exemplo n.º 19
0
// NewHandler returns a new instance of Handler.
func NewHandler(requireAuthentication bool) *Handler {
	statMap := influxdb.NewStatistics("httpd", "httpd", nil)

	config := httpd.NewConfig()
	config.AuthEnabled = requireAuthentication
	config.SharedSecret = "super secret key"

	h := &Handler{
		Handler: httpd.NewHandler(config, statMap),
	}
	h.Handler.MetaClient = &h.MetaClient
	h.Handler.QueryExecutor = influxql.NewQueryExecutor()
	h.Handler.QueryExecutor.StatementExecutor = &h.StatementExecutor
	h.Handler.QueryAuthorizer = &h.QueryAuthorizer
	h.Handler.Version = "0.0.0"
	return h
}
Exemplo n.º 20
0
// NewMeasurement allocates and initializes a new Measurement.
func NewMeasurement(name string, idx *DatabaseIndex) *Measurement {
	return &Measurement{
		Name:       name,
		fieldNames: make(map[string]struct{}),
		index:      idx,

		seriesByID:          make(map[uint64]*Series),
		seriesByTagKeyValue: make(map[string]map[string]SeriesIDs),
		seriesIDs:           make(SeriesIDs, 0),

		statMap: influxdb.NewStatistics(
			fmt.Sprintf("measurement:%s.%s", name, idx.name),
			"measurement",
			map[string]string{"database": idx.name, "measurement": name},
		),
	}
}
Exemplo n.º 21
0
// NewShard returns a new initialized Shard. walPath doesn't apply to the b1 type index
func NewShard(id uint64, index *DatabaseIndex, path string, walPath string, options EngineOptions) *Shard {
	// Configure statistics collection.
	key := fmt.Sprintf("shard:%s:%d", path, id)
	tags := map[string]string{"path": path, "id": fmt.Sprintf("%d", id), "engine": options.EngineVersion}
	statMap := influxdb.NewStatistics(key, "shard", tags)

	return &Shard{
		index:             index,
		path:              path,
		walPath:           walPath,
		id:                id,
		options:           options,
		measurementFields: make(map[string]*MeasurementFields),

		statMap:   statMap,
		LogOutput: os.Stderr,
	}
}
Exemplo n.º 22
0
// NewNodeProcessor returns a new NodeProcessor for the given node, using dir for
// the hinted-handoff data.
func NewNodeProcessor(nodeID uint64, dir string, w shardWriter, m metaClient) *NodeProcessor {
	key := strings.Join([]string{"hh_processor", dir}, ":")
	tags := map[string]string{"node": fmt.Sprintf("%d", nodeID), "path": dir}

	return &NodeProcessor{
		PurgeInterval:    DefaultPurgeInterval,
		RetryInterval:    DefaultRetryInterval,
		RetryMaxInterval: DefaultRetryMaxInterval,
		MaxSize:          DefaultMaxSize,
		MaxAge:           DefaultMaxAge,
		nodeID:           nodeID,
		dir:              dir,
		writer:           w,
		meta:             m,
		statMap:          influxdb.NewStatistics(key, "hh_processor", tags),
		Logger:           log.New(os.Stderr, "[handoff] ", log.LstdFlags),
	}
}
Exemplo n.º 23
0
func NewWAL(path string) *WAL {
	db, rp := tsdb.DecodeStorePath(path)
	return &WAL{
		path: path,

		// these options should be overriden by any options in the config
		LogOutput:   os.Stderr,
		SegmentSize: DefaultSegmentSize,
		logger:      log.New(os.Stderr, "[tsm1wal] ", log.LstdFlags),
		closing:     make(chan struct{}),

		statMap: influxdb.NewStatistics(
			"tsm1_wal:"+path,
			"tsm1_wal",
			map[string]string{"path": path, "database": db, "retentionPolicy": rp},
		),
	}
}
Exemplo n.º 24
0
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	key := strings.Join([]string{"httpd", c.BindAddress}, ":")
	tags := map[string]string{"bind": c.BindAddress}
	statMap := influxdb.NewStatistics(key, "httpd", tags)

	s := &Service{
		addr:    c.BindAddress,
		https:   c.HTTPSEnabled,
		cert:    c.HTTPSCertificate,
		limit:   c.MaxConnectionLimit,
		err:     make(chan error),
		Handler: NewHandler(c, statMap),
		Logger:  log.New(os.Stderr, "[httpd] ", log.LstdFlags),
	}
	s.Handler.Logger = s.Logger
	return s
}
Exemplo n.º 25
0
// NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory.
// Only used for engine caches, never for snapshots
func NewCache(maxSize uint64, path string) *Cache {
	db, rp := tsdb.DecodeStorePath(path)
	c := &Cache{
		maxSize: maxSize,
		store:   make(map[string]*entry),
		statMap: influxdb.NewStatistics(
			"tsm1_cache:"+path,
			"tsm1_cache",
			map[string]string{"path": path, "database": db, "retentionPolicy": rp},
		),
		lastSnapshot: time.Now(),
	}
	c.UpdateAge()
	c.UpdateCompactTime(0)
	c.updateCachedBytes(0)
	c.updateMemSize(0)
	c.updateSnapshots()
	return c
}
Exemplo n.º 26
0
func (s *Service) createSubscription(se subEntry, mode string, destinations []string) (PointsWriter, error) {
	var bm BalanceMode
	switch mode {
	case "ALL":
		bm = ALL
	case "ANY":
		bm = ANY
	default:
		return nil, fmt.Errorf("unknown balance mode %q", mode)
	}
	writers := make([]PointsWriter, len(destinations))
	statMaps := make([]*expvar.Map, len(writers))
	for i, dest := range destinations {
		u, err := url.Parse(dest)
		if err != nil {
			return nil, err
		}
		w, err := s.NewPointsWriter(*u)
		if err != nil {
			return nil, err
		}
		writers[i] = w
		tags := map[string]string{
			"database":         se.db,
			"retention_policy": se.rp,
			"name":             se.name,
			"mode":             mode,
			"destination":      dest,
		}
		key := strings.Join([]string{"subscriber", se.db, se.rp, se.name, dest}, ":")
		statMaps[i] = influxdb.NewStatistics(key, "subscriber", tags)
	}
	s.Logger.Println("created new subscription for", se.db, se.rp)
	return &balancewriter{
		bm:       bm,
		writers:  writers,
		statMaps: statMaps,
	}, nil
}
Exemplo n.º 27
0
// NewLog creates a new Log object with certain default values.
func NewLog(path string) *Log {
	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	key := strings.Join([]string{"wal", path}, ":")
	tags := map[string]string{"path": path}

	return &Log{
		path:  path,
		flush: make(chan int, 1),

		// these options should be overriden by any options in the config
		LogOutput:              os.Stderr,
		FlushColdInterval:      tsdb.DefaultFlushColdInterval,
		SegmentSize:            DefaultSegmentSize,
		MaxSeriesSize:          tsdb.DefaultMaxSeriesSize,
		CompactionThreshold:    tsdb.DefaultCompactionThreshold,
		PartitionSizeThreshold: tsdb.DefaultPartitionSizeThreshold,
		ReadySeriesSize:        tsdb.DefaultReadySeriesSize,
		flushCheckInterval:     defaultFlushCheckInterval,
		logger:                 log.New(os.Stderr, "[wal] ", log.LstdFlags),
		statMap:                influxdb.NewStatistics(key, "wal", tags),
	}
}
Exemplo n.º 28
0
// Open starts the service.
func (s *Service) Open() error {
	s.Logger.Printf("Starting collectd service")

	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	key := strings.Join([]string{"collectd", s.Config.BindAddress}, ":")
	tags := map[string]string{"bind": s.Config.BindAddress}
	s.statMap = influxdb.NewStatistics(key, "collectd", tags)

	if s.Config.BindAddress == "" {
		return fmt.Errorf("bind address is blank")
	} else if s.Config.Database == "" {
		return fmt.Errorf("database name is blank")
	} else if s.PointsWriter == nil {
		return fmt.Errorf("PointsWriter is nil")
	}

	if _, err := s.MetaClient.CreateDatabase(s.Config.Database); err != nil {
		s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Config.Database, err.Error())
		return err
	}

	if s.typesdb == nil {
		// Open collectd types.
		typesdb, err := gollectd.TypesDBFile(s.Config.TypesDB)
		if err != nil {
			return fmt.Errorf("Open(): %s", err)
		}
		s.typesdb = typesdb
	}

	// Resolve our address.
	addr, err := net.ResolveUDPAddr("udp", s.Config.BindAddress)
	if err != nil {
		return fmt.Errorf("unable to resolve UDP address: %s", err)
	}
	s.addr = addr

	// Start listening
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		return fmt.Errorf("unable to listen on UDP: %s", err)
	}

	if s.Config.ReadBuffer != 0 {
		err = conn.SetReadBuffer(s.Config.ReadBuffer)
		if err != nil {
			return fmt.Errorf("unable to set UDP read buffer to %d: %s",
				s.Config.ReadBuffer, err)
		}
	}
	s.conn = conn

	s.Logger.Println("Listening on UDP: ", conn.LocalAddr().String())

	// Start the points batcher.
	s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, s.Config.BatchPending, time.Duration(s.Config.BatchDuration))
	s.batcher.Start()

	// Create channel and wait group for signalling goroutines to stop.
	s.stop = make(chan struct{})
	s.wg.Add(2)

	// Start goroutines that process collectd packets.
	go s.serve()
	go s.writePoints()

	return nil
}