// TestBatch_MultipleBatches ensures that a batcher correctly processes multiple batches.
func TestBatch_MultipleBatches(t *testing.T) {
batchSize := 2
batcher := tsdb.NewPointBatcher(batchSize, 0, 100*time.Millisecond)
if batcher == nil {
t.Fatal("failed to create batcher for size test")
}
batcher.Start()
var p models.Point
var b []models.Point
batcher.In() <- p
batcher.In() <- p
b = <-batcher.Out() // Batch threshold reached.
if len(b) != batchSize {
t.Errorf("received batch (size) has incorrect length exp %d, got %d", batchSize, len(b))
}
batcher.In() <- p
b = <-batcher.Out() // Timeout triggered.
if len(b) != 1 {
t.Errorf("received batch (timeout) has incorrect length exp %d, got %d", 1, len(b))
}
checkPointBatcherStats(t, batcher, -1, 3, 1, 1)
}
// Open starts the service
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
s.Logger.Println("Starting OpenTSDB service")
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
key := strings.Join([]string{"opentsdb", s.BindAddress}, ":")
tags := map[string]string{"bind": s.BindAddress}
s.statMap = influxdb.NewStatistics(key, "opentsdb", tags)
if _, err := s.MetaClient.CreateDatabase(s.Database); err != nil {
s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Database, err.Error())
return err
}
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go s.processBatches(s.batcher)
// Open listener.
if s.tls {
cert, err := tls.LoadX509KeyPair(s.cert, s.cert)
if err != nil {
return err
}
listener, err := tls.Listen("tcp", s.BindAddress, &tls.Config{
Certificates: []tls.Certificate{cert},
})
if err != nil {
return err
}
s.Logger.Println("Listening on TLS:", listener.Addr().String())
s.ln = listener
} else {
listener, err := net.Listen("tcp", s.BindAddress)
if err != nil {
return err
}
s.Logger.Println("Listening on:", listener.Addr().String())
s.ln = listener
}
s.httpln = newChanListener(s.ln.Addr())
// Begin listening for connections.
s.wg.Add(2)
go s.serveHTTP()
go s.serve()
return nil
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
d := *c.WithDefaults()
return &Service{
config: d,
done: make(chan struct{}),
parserChan: make(chan []byte, parserChanLen),
batcher: tsdb.NewPointBatcher(d.BatchSize, d.BatchPending, time.Duration(d.BatchTimeout)),
Logger: log.New(os.Stderr, "[udp] ", log.LstdFlags),
}
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
d := *c.WithDefaults()
return &Service{
config: d,
parserChan: make(chan []byte, parserChanLen),
batcher: tsdb.NewPointBatcher(d.BatchSize, d.BatchPending, time.Duration(d.BatchTimeout)),
Logger: zap.New(zap.NullEncoder()),
stats: &Statistics{},
defaultTags: models.StatisticTags{"bind": d.BindAddress},
}
}
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
tags := map[string]string{"proto": s.protocol, "bind": s.bindAddress}
s.statMap = influxdb.NewStatistics(s.diagsKey, "graphite", tags)
// Register diagnostics if a Monitor service is available.
if s.Monitor != nil {
s.Monitor.RegisterDiagnosticsClient(s.diagsKey, s)
}
if db := s.MetaClient.Database(s.database); db != nil {
if rp, _ := s.MetaClient.RetentionPolicy(s.database, s.retentionPolicy); rp == nil {
rpi := meta.NewRetentionPolicyInfo(s.retentionPolicy)
if _, err := s.MetaClient.CreateRetentionPolicy(s.database, rpi); err != nil {
s.logger.Printf("Failed to ensure target retention policy %s exists: %s", s.database, err.Error())
}
}
} else {
rpi := meta.NewRetentionPolicyInfo(s.retentionPolicy)
if _, err := s.MetaClient.CreateDatabaseWithRetentionPolicy(s.database, rpi); err != nil {
s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
return err
}
}
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go s.processBatches(s.batcher)
var err error
if strings.ToLower(s.protocol) == "tcp" {
s.addr, err = s.openTCPServer()
} else if strings.ToLower(s.protocol) == "udp" {
s.addr, err = s.openUDPServer()
} else {
return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
}
if err != nil {
return err
}
s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
return nil
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
d := *c.WithDefaults()
return &Service{
config: d,
done: make(chan struct{}),
parserChan: make(chan []byte, parserChanLen),
batcher: tsdb.NewPointBatcher(d.BatchSize, d.BatchPending, time.Duration(d.BatchTimeout)),
Logger: log.New(os.Stderr, "[udp] ", log.LstdFlags),
stats: &Statistics{},
defaultTags: models.StatisticTags{"bind": d.BindAddress},
}
}
// Open starts the service
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.closed() {
return nil // Already open.
}
s.done = make(chan struct{})
s.Logger.Info("Starting OpenTSDB service")
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go func() { defer s.wg.Done(); s.processBatches(s.batcher) }()
// Open listener.
if s.tls {
cert, err := tls.LoadX509KeyPair(s.cert, s.cert)
if err != nil {
return err
}
listener, err := tls.Listen("tcp", s.BindAddress, &tls.Config{
Certificates: []tls.Certificate{cert},
})
if err != nil {
return err
}
s.Logger.Info(fmt.Sprint("Listening on TLS: ", listener.Addr().String()))
s.ln = listener
} else {
listener, err := net.Listen("tcp", s.BindAddress)
if err != nil {
return err
}
s.Logger.Info(fmt.Sprint("Listening on: ", listener.Addr().String()))
s.ln = listener
}
s.httpln = newChanListener(s.ln.Addr())
// Begin listening for connections.
s.wg.Add(2)
go func() { defer s.wg.Done(); s.serve() }()
go func() { defer s.wg.Done(); s.serveHTTP() }()
return nil
}
// Open starts the service
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
s.Logger.Println("Starting OpenTSDB service")
if _, err := s.MetaClient.CreateDatabase(s.Database); err != nil {
s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Database, err.Error())
return err
}
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go s.processBatches(s.batcher)
// Open listener.
if s.tls {
cert, err := tls.LoadX509KeyPair(s.cert, s.cert)
if err != nil {
return err
}
listener, err := tls.Listen("tcp", s.BindAddress, &tls.Config{
Certificates: []tls.Certificate{cert},
})
if err != nil {
return err
}
s.Logger.Println("Listening on TLS:", listener.Addr().String())
s.ln = listener
} else {
listener, err := net.Listen("tcp", s.BindAddress)
if err != nil {
return err
}
s.Logger.Println("Listening on:", listener.Addr().String())
s.ln = listener
}
s.httpln = newChanListener(s.ln.Addr())
// Begin listening for connections.
s.wg.Add(2)
go s.serveHTTP()
go s.serve()
return nil
}
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)
// Register diagnostics if a Monitor service is available.
if s.Monitor != nil {
s.Monitor.RegisterDiagnosticsClient(s.diagsKey, s)
}
if db := s.MetaClient.Database(s.database); db != nil {
if rp, _ := s.MetaClient.RetentionPolicy(s.database, s.retentionPolicy); rp == nil {
spec := meta.RetentionPolicySpec{Name: s.retentionPolicy}
if _, err := s.MetaClient.CreateRetentionPolicy(s.database, &spec); err != nil {
s.logger.Printf("Failed to ensure target retention policy %s exists: %s", s.database, err.Error())
}
}
} else {
spec := meta.RetentionPolicySpec{Name: s.retentionPolicy}
if _, err := s.MetaClient.CreateDatabaseWithRetentionPolicy(s.database, &spec); err != nil {
s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
return err
}
}
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go s.processBatches(s.batcher)
var err error
if strings.ToLower(s.protocol) == "tcp" {
s.addr, err = s.openTCPServer()
} else if strings.ToLower(s.protocol) == "udp" {
s.addr, err = s.openUDPServer()
} else {
return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
}
if err != nil {
return err
}
s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
return nil
}
// TestBatch_Flush ensures that a batcher generates a batch when flushed
func TestBatch_Flush(t *testing.T) {
batchSize := 2
batcher := tsdb.NewPointBatcher(batchSize, 0, time.Hour)
if batcher == nil {
t.Fatal("failed to create batcher for flush test")
}
batcher.Start()
var p models.Point
go func() {
batcher.In() <- p
batcher.Flush()
}()
batch := <-batcher.Out()
if len(batch) != 1 {
t.Errorf("received batch has incorrect length exp %d, got %d", 1, len(batch))
}
checkPointBatcherStats(t, batcher, -1, 1, 0, 0)
}
// TestBatch_Size ensures that a batcher generates a batch when the timeout triggers.
func TestBatch_Timeout(t *testing.T) {
batchSize := 5
batcher := tsdb.NewPointBatcher(batchSize+1, 0, 100*time.Millisecond)
if batcher == nil {
t.Fatal("failed to create batcher for timeout test")
}
batcher.Start()
var p models.Point
go func() {
for i := 0; i < batchSize; i++ {
batcher.In() <- p
}
}()
batch := <-batcher.Out()
if len(batch) != batchSize {
t.Errorf("received batch has incorrect length exp %d, got %d", batchSize, len(batch))
}
checkPointBatcherStats(t, batcher, -1, batchSize, 0, 1)
}
// TestBatch_Size ensures that a buffered batcher generates a batch when the size threshold is reached.
func TestBatch_SizeBuffered(t *testing.T) {
batchSize := 5
batcher := tsdb.NewPointBatcher(batchSize, 5, time.Hour)
if batcher == nil {
t.Fatal("failed to create batcher for size test")
}
batcher.Start()
var p models.Point
go func() {
for i := 0; i < batchSize; i++ {
batcher.In() <- p
}
}()
batch := <-batcher.Out()
if len(batch) != batchSize {
t.Errorf("received batch has incorrect length exp %d, got %d", batchSize, len(batch))
}
checkPointBatcherStats(t, batcher, -1, batchSize, 1, 0)
}
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.closed() {
return nil // Already open.
}
s.done = make(chan struct{})
s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)
// Register diagnostics if a Monitor service is available.
if s.Monitor != nil {
s.Monitor.RegisterDiagnosticsClient(s.diagsKey, s)
}
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go s.processBatches(s.batcher)
var err error
if strings.ToLower(s.protocol) == "tcp" {
s.addr, err = s.openTCPServer()
} else if strings.ToLower(s.protocol) == "udp" {
s.addr, err = s.openUDPServer()
} else {
return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
}
if err != nil {
return err
}
s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
return nil
}
// Open starts the service.
func (s *Service) Open() error {
s.Logger.Printf("Starting collectd service")
// Configure expvar monitoring. It's OK to do this even if the service fails to open and
// should be done before any data could arrive for the service.
key := strings.Join([]string{"collectd", s.Config.BindAddress}, ":")
tags := map[string]string{"bind": s.Config.BindAddress}
s.statMap = influxdb.NewStatistics(key, "collectd", tags)
if s.Config.BindAddress == "" {
return fmt.Errorf("bind address is blank")
} else if s.Config.Database == "" {
return fmt.Errorf("database name is blank")
} else if s.PointsWriter == nil {
return fmt.Errorf("PointsWriter is nil")
}
if _, err := s.MetaClient.CreateDatabase(s.Config.Database); err != nil {
s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Config.Database, err.Error())
return err
}
if s.typesdb == nil {
// Open collectd types.
typesdb, err := gollectd.TypesDBFile(s.Config.TypesDB)
if err != nil {
return fmt.Errorf("Open(): %s", err)
}
s.typesdb = typesdb
}
// Resolve our address.
addr, err := net.ResolveUDPAddr("udp", s.Config.BindAddress)
if err != nil {
return fmt.Errorf("unable to resolve UDP address: %s", err)
}
s.addr = addr
// Start listening
conn, err := net.ListenUDP("udp", addr)
if err != nil {
return fmt.Errorf("unable to listen on UDP: %s", err)
}
if s.Config.ReadBuffer != 0 {
err = conn.SetReadBuffer(s.Config.ReadBuffer)
if err != nil {
return fmt.Errorf("unable to set UDP read buffer to %d: %s",
s.Config.ReadBuffer, err)
}
}
s.conn = conn
s.Logger.Println("Listening on UDP: ", conn.LocalAddr().String())
// Start the points batcher.
s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, s.Config.BatchPending, time.Duration(s.Config.BatchDuration))
s.batcher.Start()
// Create channel and wait group for signalling goroutines to stop.
s.stop = make(chan struct{})
s.wg.Add(2)
// Start goroutines that process collectd packets.
go s.serve()
go s.writePoints()
return nil
}
// Open starts the service.
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.closed() {
return nil // Already open.
}
s.done = make(chan struct{})
s.Logger.Printf("Starting collectd service")
if s.Config.BindAddress == "" {
return fmt.Errorf("bind address is blank")
} else if s.Config.Database == "" {
return fmt.Errorf("database name is blank")
} else if s.PointsWriter == nil {
return fmt.Errorf("PointsWriter is nil")
}
if s.typesdb == nil {
// Open collectd types.
if stat, err := os.Stat(s.Config.TypesDB); err != nil {
return fmt.Errorf("Stat(): %s", err)
} else if stat.IsDir() {
alltypesdb := make(gollectd.Types)
var readdir func(path string)
readdir = func(path string) {
files, err := ioutil.ReadDir(path)
if err != nil {
s.Logger.Printf("Unable to read directory %s: %s\n", path, err)
return
}
for _, f := range files {
fullpath := filepath.Join(path, f.Name())
if f.IsDir() {
readdir(fullpath)
continue
}
s.Logger.Printf("Loading %s\n", fullpath)
types, err := gollectd.TypesDBFile(fullpath)
if err != nil {
s.Logger.Printf("Unable to parse collectd types file: %s\n", f.Name())
continue
}
for k, t := range types {
a, ok := alltypesdb[k]
if ok {
alltypesdb[k] = t
} else {
alltypesdb[k] = append(a, t...)
}
}
}
}
readdir(s.Config.TypesDB)
s.typesdb = alltypesdb
} else {
s.Logger.Printf("Loading %s\n", s.Config.TypesDB)
typesdb, err := gollectd.TypesDBFile(s.Config.TypesDB)
if err != nil {
return fmt.Errorf("Open(): %s", err)
}
s.typesdb = typesdb
}
}
// Resolve our address.
addr, err := net.ResolveUDPAddr("udp", s.Config.BindAddress)
if err != nil {
return fmt.Errorf("unable to resolve UDP address: %s", err)
}
s.addr = addr
// Start listening
conn, err := net.ListenUDP("udp", addr)
if err != nil {
return fmt.Errorf("unable to listen on UDP: %s", err)
}
if s.Config.ReadBuffer != 0 {
err = conn.SetReadBuffer(s.Config.ReadBuffer)
if err != nil {
return fmt.Errorf("unable to set UDP read buffer to %d: %s",
s.Config.ReadBuffer, err)
}
}
s.conn = conn
s.Logger.Println("Listening on UDP: ", conn.LocalAddr().String())
// Start the points batcher.
s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, s.Config.BatchPending, time.Duration(s.Config.BatchDuration))
s.batcher.Start()
// Create waitgroup for signalling goroutines to stop and start goroutines
// that process collectd packets.
s.wg.Add(2)
go func() { defer s.wg.Done(); s.serve() }()
go func() { defer s.wg.Done(); s.writePoints() }()
return nil
}
// Open starts the service.
func (s *Service) Open() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.closed() {
return nil // Already open.
}
s.done = make(chan struct{})
s.Logger.Info("Starting collectd service")
if s.Config.BindAddress == "" {
return fmt.Errorf("bind address is blank")
} else if s.Config.Database == "" {
return fmt.Errorf("database name is blank")
} else if s.PointsWriter == nil {
return fmt.Errorf("PointsWriter is nil")
}
if s.popts.TypesDB == nil {
// Open collectd types.
if stat, err := os.Stat(s.Config.TypesDB); err != nil {
return fmt.Errorf("Stat(): %s", err)
} else if stat.IsDir() {
alltypesdb := new(api.TypesDB)
var readdir func(path string)
readdir = func(path string) {
files, err := ioutil.ReadDir(path)
if err != nil {
s.Logger.Info(fmt.Sprintf("Unable to read directory %s: %s\n", path, err))
return
}
for _, f := range files {
fullpath := filepath.Join(path, f.Name())
if f.IsDir() {
readdir(fullpath)
continue
}
s.Logger.Info(fmt.Sprintf("Loading %s\n", fullpath))
types, err := TypesDBFile(fullpath)
if err != nil {
s.Logger.Info(fmt.Sprintf("Unable to parse collectd types file: %s\n", f.Name()))
continue
}
alltypesdb.Merge(types)
}
}
readdir(s.Config.TypesDB)
s.popts.TypesDB = alltypesdb
} else {
s.Logger.Info(fmt.Sprintf("Loading %s\n", s.Config.TypesDB))
types, err := TypesDBFile(s.Config.TypesDB)
if err != nil {
return fmt.Errorf("Open(): %s", err)
}
s.popts.TypesDB = types
}
}
// Sets the security level according to the config.
// Default not necessary because we validate the config.
switch s.Config.SecurityLevel {
case "none":
s.popts.SecurityLevel = network.None
case "sign":
s.popts.SecurityLevel = network.Sign
case "encrypt":
s.popts.SecurityLevel = network.Encrypt
}
// Sets the auth file according to the config.
if s.popts.PasswordLookup == nil {
s.popts.PasswordLookup = network.NewAuthFile(s.Config.AuthFile)
}
// Resolve our address.
addr, err := net.ResolveUDPAddr("udp", s.Config.BindAddress)
if err != nil {
return fmt.Errorf("unable to resolve UDP address: %s", err)
}
s.addr = addr
// Start listening
conn, err := net.ListenUDP("udp", addr)
if err != nil {
return fmt.Errorf("unable to listen on UDP: %s", err)
}
if s.Config.ReadBuffer != 0 {
err = conn.SetReadBuffer(s.Config.ReadBuffer)
if err != nil {
return fmt.Errorf("unable to set UDP read buffer to %d: %s",
s.Config.ReadBuffer, err)
}
}
s.conn = conn
s.Logger.Info(fmt.Sprint("Listening on UDP: ", conn.LocalAddr().String()))
// Start the points batcher.
s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, s.Config.BatchPending, time.Duration(s.Config.BatchDuration))
s.batcher.Start()
// Create waitgroup for signalling goroutines to stop and start goroutines
// that process collectd packets.
s.wg.Add(2)
go func() { defer s.wg.Done(); s.serve() }()
go func() { defer s.wg.Done(); s.writePoints() }()
return nil
}