Example #1
0
func (s *Service) Open() (err error) {
	if s.config.BindAddress == "" {
		return errors.New("bind address has to be specified in config")
	}
	if s.config.Database == "" {
		return errors.New("database has to be specified in config")
	}

	s.addr, err = net.ResolveUDPAddr("udp", s.config.BindAddress)
	if err != nil {
		s.Logger.Printf("Failed to resolve UDP address %s: %s", s.config.BindAddress, err)
		return err
	}

	s.conn, err = net.ListenUDP("udp", s.addr)
	if err != nil {
		s.Logger.Printf("Failed to set up UDP listener at address %s: %s", s.addr, err)
		return err
	}

	s.Logger.Printf("Started listening on %s", s.config.BindAddress)

	s.batcher = tsdb.NewPointBatcher(s.config.BatchSize, time.Duration(s.config.BatchTimeout))

	s.wg.Add(2)
	go s.serve()
	go s.writePoints()

	return nil
}
Example #2
0
// Open starts the service
func (s *Service) Open() error {
	s.Logger.Println("Starting OpenTSDB service")

	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	key := strings.Join([]string{"opentsdb", s.BindAddress}, ":")
	tags := map[string]string{"bind": s.BindAddress}
	s.statMap = influxdb.NewStatistics(key, "opentsdb", tags)

	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.Logger.Printf("Failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.Database); err != nil {
		s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Database, err.Error())
		return err
	}

	s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
	s.batcher.Start()

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(s.batcher)

	// Open listener.
	if s.tls {
		cert, err := tls.LoadX509KeyPair(s.cert, s.cert)
		if err != nil {
			return err
		}

		listener, err := tls.Listen("tcp", s.BindAddress, &tls.Config{
			Certificates: []tls.Certificate{cert},
		})
		if err != nil {
			return err
		}

		s.Logger.Println("Listening on TLS:", listener.Addr().String())
		s.ln = listener
	} else {
		listener, err := net.Listen("tcp", s.BindAddress)
		if err != nil {
			return err
		}

		s.Logger.Println("Listening on:", listener.Addr().String())
		s.ln = listener
	}
	s.httpln = newChanListener(s.ln.Addr())

	// Begin listening for connections.
	s.wg.Add(2)
	go s.serveHTTP()
	go s.serve()

	return nil
}
Example #3
0
// Open starts the service.
func (s *Service) Open() error {
	s.Logger.Printf("Starting collectd service")

	if s.Config.BindAddress == "" {
		return fmt.Errorf("bind address is blank")
	} else if s.Config.Database == "" {
		return fmt.Errorf("database name is blank")
	} else if s.PointsWriter == nil {
		return fmt.Errorf("PointsWriter is nil")
	}

	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.Logger.Printf("Failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.Config.Database); err != nil {
		s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Config.Database, err.Error())
		return err
	}

	if s.typesdb == nil {
		// Open collectd types.
		typesdb, err := gollectd.TypesDBFile(s.Config.TypesDB)
		if err != nil {
			return fmt.Errorf("Open(): %s", err)
		}
		s.typesdb = typesdb
	}

	// Resolve our address.
	addr, err := net.ResolveUDPAddr("udp", s.Config.BindAddress)
	if err != nil {
		return fmt.Errorf("unable to resolve UDP address: %s", err)
	}
	s.addr = addr

	// Start listening
	ln, err := net.ListenUDP("udp", addr)
	if err != nil {
		return fmt.Errorf("unable to listen on UDP: %s", err)
	}
	s.ln = ln

	s.Logger.Println("Listening on UDP: ", ln.LocalAddr().String())

	// Start the points batcher.
	s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, time.Duration(s.Config.BatchDuration))
	s.batcher.Start()

	// Create channel and wait group for signalling goroutines to stop.
	s.stop = make(chan struct{})
	s.wg.Add(2)

	// Start goroutines that process collectd packets.
	go s.serve()
	go s.writePoints()

	return nil
}
Example #4
0
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
	s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)

	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.logger.Printf("Failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.database); err != nil {
		s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
		return err
	}

	s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchTimeout)
	s.batcher.Start()

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(s.batcher)

	var err error
	if strings.ToLower(s.protocol) == "tcp" {
		s.addr, err = s.openTCPServer()
	} else if strings.ToLower(s.protocol) == "udp" {
		s.addr, err = s.openUDPServer()
	} else {
		return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
	}
	if err != nil {
		return err
	}

	s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
	return nil
}
Example #5
0
// handleTCPConnection services an individual TCP connection for the Graphite input.
func (s *Service) handleTCPConnection(conn net.Conn) {
	defer conn.Close()
	defer s.wg.Done()

	batcher := tsdb.NewPointBatcher(s.batchSize, s.batchTimeout)
	batcher.Start()
	reader := bufio.NewReader(conn)

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(batcher)

	for {
		// Read up to the next newline.
		buf, err := reader.ReadBytes('\n')
		if err != nil {
			batcher.Flush()
			return
		}

		// Trim the buffer, even though there should be no padding
		line := strings.TrimSpace(string(buf))

		// Parse it.
		point, err := s.parser.Parse(line)
		if err != nil {
			s.logger.Printf("unable to parse data: %s", err)
			continue
		}
		batcher.In() <- point
	}
}
Example #6
0
// TestBatch_MultipleBatches ensures that a batcher correctly processes multiple batches.
func TestBatch_MultipleBatches(t *testing.T) {
	batchSize := 2
	batcher := tsdb.NewPointBatcher(batchSize, 0, 100*time.Millisecond)
	if batcher == nil {
		t.Fatal("failed to create batcher for size test")
	}

	batcher.Start()

	var p tsdb.Point
	var b []tsdb.Point

	batcher.In() <- p
	batcher.In() <- p
	b = <-batcher.Out() // Batch threshold reached.
	if len(b) != batchSize {
		t.Errorf("received batch (size) has incorrect length exp %d, got %d", batchSize, len(b))
	}

	batcher.In() <- p
	b = <-batcher.Out() // Timeout triggered.
	if len(b) != 1 {
		t.Errorf("received batch (timeout) has incorrect length exp %d, got %d", 1, len(b))
	}

	checkPointBatcherStats(t, batcher, -1, 3, 1, 1)
}
Example #7
0
func NewService(c Config) *Service {
	return &Service{
		config:  c,
		done:    make(chan struct{}),
		batcher: tsdb.NewPointBatcher(c.BatchSize, time.Duration(c.BatchTimeout)),
		Logger:  log.New(os.Stderr, "[udp] ", log.LstdFlags),
	}
}
Example #8
0
func NewService(c Config) *Service {
	d := *c.WithDefaults()
	return &Service{
		config:  d,
		done:    make(chan struct{}),
		batcher: tsdb.NewPointBatcher(d.BatchSize, d.BatchPending, time.Duration(d.BatchTimeout)),
		Logger:  log.New(os.Stderr, "[udp] ", log.LstdFlags),
	}
}
Example #9
0
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
	s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)

	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	s.setExpvar()

	// // One Graphite service hooks up diagnostics for all Graphite functionality.
	monitorOnce.Do(func() {
		if s.Monitor == nil {
			s.logger.Println("no monitor service available, no monitoring will be performed")
			return
		}
		s.Monitor.RegisterDiagnosticsClient("graphite", monitor.DiagsClientFunc(handleDiagnostics))
	})

	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.logger.Printf("Failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.database); err != nil {
		s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
		return err
	}

	s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchTimeout)
	s.batcher.Start()

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(s.batcher)

	var err error
	if strings.ToLower(s.protocol) == "tcp" {
		s.addr, err = s.openTCPServer()
	} else if strings.ToLower(s.protocol) == "udp" {
		s.addr, err = s.openUDPServer()
	} else {
		return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
	}
	if err != nil {
		return err
	}

	// Register stats for this service, now that it has started successfully.
	if s.Monitor != nil {
		t := monitor.NewStatsMonitorClient(s.statMap)
		s.Monitor.RegisterStatsClient("graphite",
			map[string]string{"proto": s.protocol, "bind": s.bindAddress}, t)
	}

	s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
	return nil
}
Example #10
0
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
	s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)

	// One Graphite service hooks up monitoring for all Graphite functionality.
	monitorOnce.Do(func() {
		if s.Monitor == nil {
			s.logger.Println("no monitor service available, no monitoring will be performed")
			return
		}

		t := monitor.NewMonitorClient(statMapTCP)
		s.Monitor.Register("graphite", map[string]string{"proto": "tcp"}, t)

		u := monitor.NewMonitorClient(statMapUDP)
		s.Monitor.Register("graphite", map[string]string{"proto": "udp"}, u)
	})

	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.logger.Printf("Failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.database); err != nil {
		s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
		return err
	}

	s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchTimeout)
	s.batcher.Start()

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(s.batcher)

	var err error
	if strings.ToLower(s.protocol) == "tcp" {
		s.addr, err = s.openTCPServer()
	} else if strings.ToLower(s.protocol) == "udp" {
		s.addr, err = s.openUDPServer()
	} else {
		return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
	}
	if err != nil {
		return err
	}

	s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
	return nil
}
Example #11
0
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)

	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	key := strings.Join([]string{"graphite", s.protocol, s.bindAddress}, ":")
	tags := map[string]string{"proto": s.protocol, "bind": s.bindAddress}
	s.statMap = influxdb.NewStatistics(key, "graphite", tags)

	// Register diagnostics if a Monitor service is available.
	if s.Monitor != nil {
		s.Monitor.RegisterDiagnosticsClient(key, s)
	}

	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.logger.Printf("Failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.database); err != nil {
		s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
		return err
	}

	s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchPending, s.batchTimeout)
	s.batcher.Start()

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(s.batcher)

	var err error
	if strings.ToLower(s.protocol) == "tcp" {
		s.addr, err = s.openTCPServer()
	} else if strings.ToLower(s.protocol) == "udp" {
		s.addr, err = s.openUDPServer()
	} else {
		return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
	}
	if err != nil {
		return err
	}

	s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
	return nil
}
Example #12
0
// TestBatch_Flush ensures that a batcher generates a batch when flushed
func TestBatch_Flush(t *testing.T) {
	batchSize := 2
	batcher := tsdb.NewPointBatcher(batchSize, 0, time.Hour)
	if batcher == nil {
		t.Fatal("failed to create batcher for flush test")
	}

	batcher.Start()

	var p tsdb.Point
	go func() {
		batcher.In() <- p
		batcher.Flush()
	}()
	batch := <-batcher.Out()
	if len(batch) != 1 {
		t.Errorf("received batch has incorrect length exp %d, got %d", 1, len(batch))
	}
	checkPointBatcherStats(t, batcher, -1, 1, 0, 0)
}
Example #13
0
// TestBatch_Size ensures that a batcher generates a batch when the timeout triggers.
func TestBatch_Timeout(t *testing.T) {
	batchSize := 5
	batcher := tsdb.NewPointBatcher(batchSize+1, 0, 100*time.Millisecond)
	if batcher == nil {
		t.Fatal("failed to create batcher for timeout test")
	}

	batcher.Start()

	var p tsdb.Point
	go func() {
		for i := 0; i < batchSize; i++ {
			batcher.In() <- p
		}
	}()
	batch := <-batcher.Out()
	if len(batch) != batchSize {
		t.Errorf("received batch has incorrect length exp %d, got %d", batchSize, len(batch))
	}
	checkPointBatcherStats(t, batcher, -1, batchSize, 0, 1)
}
Example #14
0
// TestBatch_Size ensures that a buffered batcher generates a batch when the size threshold is reached.
func TestBatch_SizeBuffered(t *testing.T) {
	batchSize := 5
	batcher := tsdb.NewPointBatcher(batchSize, 5, time.Hour)
	if batcher == nil {
		t.Fatal("failed to create batcher for size test")
	}

	batcher.Start()

	var p tsdb.Point
	go func() {
		for i := 0; i < batchSize; i++ {
			batcher.In() <- p
		}
	}()
	batch := <-batcher.Out()
	if len(batch) != batchSize {
		t.Errorf("received batch has incorrect length exp %d, got %d", batchSize, len(batch))
	}
	checkPointBatcherStats(t, batcher, -1, batchSize, 1, 0)
}
Example #15
0
// openUDPServer opens the Graphite input in UDP mode and starts processing incoming data.
func (s *Service) openUDPServer() (net.Addr, error) {
	addr, err := net.ResolveUDPAddr("udp", s.bindAddress)
	if err != nil {
		return nil, err
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		return nil, err
	}

	batcher := tsdb.NewPointBatcher(s.batchSize, s.batchTimeout)
	batcher.Start()

	// Start processing batches.
	s.wg.Add(1)
	go s.processBatches(batcher)

	buf := make([]byte, udpBufferSize)
	s.wg.Add(1)
	go func() {
		defer s.wg.Done()
		for {
			n, _, err := conn.ReadFromUDP(buf)
			if err != nil {
				batcher.Flush()
				conn.Close()
				return
			}
			for _, line := range strings.Split(string(buf[:n]), "\n") {
				point, err := s.parser.Parse(line)
				if err != nil {
					continue
				}
				batcher.In() <- point
			}
		}
	}()
	return conn.LocalAddr(), nil
}
Example #16
0
// Open starts the service.
func (s *Service) Open() error {
	s.Logger.Printf("Starting collectd service")

	// Configure expvar monitoring. It's OK to do this even if the service fails to open and
	// should be done before any data could arrive for the service.
	key := strings.Join([]string{"collectd", s.Config.BindAddress}, ":")
	tags := map[string]string{"bind": s.Config.BindAddress}
	s.statMap = influxdb.NewStatistics(key, "collectd", tags)

	if s.Config.BindAddress == "" {
		return fmt.Errorf("bind address is blank")
	} else if s.Config.Database == "" {
		return fmt.Errorf("database name is blank")
	} else if s.PointsWriter == nil {
		return fmt.Errorf("PointsWriter is nil")
	}

	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.Logger.Printf("Failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.Config.Database); err != nil {
		s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Config.Database, err.Error())
		return err
	}

	if s.typesdb == nil {
		// Open collectd types.
		typesdb, err := gollectd.TypesDBFile(s.Config.TypesDB)
		if err != nil {
			return fmt.Errorf("Open(): %s", err)
		}
		s.typesdb = typesdb
	}

	// Resolve our address.
	addr, err := net.ResolveUDPAddr("udp", s.Config.BindAddress)
	if err != nil {
		return fmt.Errorf("unable to resolve UDP address: %s", err)
	}
	s.addr = addr

	// Start listening
	ln, err := net.ListenUDP("udp", addr)
	if err != nil {
		return fmt.Errorf("unable to listen on UDP: %s", err)
	}
	s.ln = ln

	s.Logger.Println("Listening on UDP: ", ln.LocalAddr().String())

	// Start the points batcher.
	s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, s.Config.BatchPending, time.Duration(s.Config.BatchDuration))
	s.batcher.Start()

	// Create channel and wait group for signalling goroutines to stop.
	s.stop = make(chan struct{})
	s.wg.Add(2)

	// Start goroutines that process collectd packets.
	go s.serve()
	go s.writePoints()

	return nil
}
Example #17
0
// Open starts the service.
func (s *Service) Open() error {
	if s.Config.AMQPAddress == "" {
		return fmt.Errorf("AMQP address is blank")
	} else if s.Config.AMQPPort == "" {
		return fmt.Errorf("AMQP port is blank")
	} else if s.PointsWriter == nil {
		return fmt.Errorf("PointsWriter is nil")
	}

	if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
		s.Logger.Printf("failed to detect a cluster leader: %s", err.Error())
		return err
	}

	if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.Config.Database); err != nil {
		s.Logger.Printf("failed to ensure target database %s exists: %s", s.Config.Database, err.Error())
		return err
	}

	connectUrl := fmt.Sprintf("amqp://%s:%s@%s:%s/%s", s.Config.User, s.Config.Pass, s.Config.AMQPAddress, s.Config.AMQPPort, s.Config.VHost)
	s.conn, err = amqp.Dial(connectUrl)
	if err != nil {
		s.Logger.Printf("could not connect to AMQP server: %s", err)
	}

	s.c, err = s.conn.Channel()
	if err != nil {
		s.Logger.Printf("could not create channel on AMQP server: %s", err)
	}

	err = c.ExchangeDeclare(s.Config.Exchange, "topic", true, false, false, false, nil)
	if err != nil {
		s.Logger.Printf("exchange.declare: %s", err)
	}

	_, err = c.QueueDeclare(s.Config.Queue, true, false, false, false, nil)
	if err != nil {
		s.Logger.Printf("queue.declare: %v", err)
	}

	err = c.QueueBind(s.Config.Queue, s.Config.Key, s.Config.Exchange, false, nil)
	if err != nil {
		s.Logger.Printf("queue.bind: %v", err)
	}

	s.messages, err = s.c.Consume(s.Config.Queue, "InfluxDB", false, false, false, false, nil)
	if err != nil {
		s.Logger.Printf("queue.consume: %v", err)
	}

	// Start the points batcher.
	s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, time.Duration(s.Config.BatchDuration))
	s.batcher.Start()

	// Create channel and wait group for signalling goroutines to stop.
	s.stop = make(chan struct{})
	s.wg.Add(2)

	// Start goroutines that process collectd packets.
	go s.serve()
	go s.writePoints()

	s.Logger.Println("AMQP listener started")

	return nil
}