Example #1
0
func (b *Beat) launch(bt Creator) error {
	err := b.handleFlags()
	if err != nil {
		return err
	}

	svc.BeforeRun()
	defer svc.Cleanup()

	if err := b.configure(); err != nil {
		return err
	}

	// load the beats config section
	var sub *common.Config
	configName := strings.ToLower(b.Name)
	if b.RawConfig.HasField(configName) {
		sub, err = b.RawConfig.Child(configName, -1)
		if err != nil {
			return err
		}
	} else {
		sub = common.NewConfig()
	}

	logp.Info("Setup Beat: %s; Version: %s", b.Name, b.Version)
	processors, err := processors.New(b.Config.Processors)
	if err != nil {
		return fmt.Errorf("error initializing processors: %v", err)
	}

	debugf("Initializing output plugins")
	publisher, err := publisher.New(b.Name, b.Config.Output, b.Config.Shipper, processors)
	if err != nil {
		return fmt.Errorf("error initializing publisher: %v", err)
	}

	// TODO: some beats race on shutdown with publisher.Stop -> do not call Stop yet,
	//       but refine publisher to disconnect clients on stop automatically
	// defer publisher.Stop()

	b.Publisher = publisher
	beater, err := bt(b, sub)
	if err != nil {
		return err
	}

	// If -configtest was specified, exit now prior to run.
	if cfgfile.IsTestConfig() {
		fmt.Println("Config OK")
		return GracefulExit
	}

	svc.HandleSignals(beater.Stop)

	logp.Info("%s start running.", b.Name)
	defer logp.Info("%s stopped.", b.Name)

	return beater.Run(b)
}
Example #2
0
func (c *Crawler) Start(states file.States) error {

	logp.Info("Loading Prospectors: %v", len(c.prospectorConfigs))

	// Prospect the globs/paths given on the command line and launch harvesters
	for _, prospectorConfig := range c.prospectorConfigs {

		prospector, err := prospector.NewProspector(prospectorConfig, states, c.spooler.Channel)
		if err != nil {
			return fmt.Errorf("Error in initing prospector: %s", err)
		}
		c.prospectors = append(c.prospectors, prospector)
	}

	logp.Info("Loading Prospectors completed. Number of prospectors: %v", len(c.prospectors))

	for i, p := range c.prospectors {
		c.wg.Add(1)

		go func(id int, prospector *prospector.Prospector) {
			defer func() {
				c.wg.Done()
				logp.Debug("crawler", "Prospector %v stopped", id)
			}()
			logp.Debug("crawler", "Starting prospector %v", id)
			prospector.Run()
		}(i, p)
	}

	logp.Info("All prospectors are initialised and running with %d states to persist", states.Count())

	return nil
}
Example #3
0
func (r *Registrar) fetchState(filePath string, fileInfo os.FileInfo) (int64, bool) {

	// Check if there is a state for this file
	lastState, isFound := r.GetFileState(filePath)

	if isFound && input.IsSameFile(filePath, fileInfo) {
		logp.Debug("registar", "Same file as before found. Fetch the state and persist it.")
		// We're resuming - throw the last state back downstream so we resave it
		// And return the offset - also force harvest in case the file is old and we're about to skip it
		r.Persist <- lastState
		return lastState.Offset, true
	}

	if previous, err := r.getPreviousFile(filePath, fileInfo); err == nil {
		// File has rotated between shutdown and startup
		// We return last state downstream, with a modified event source with the new file name
		// And return the offset - also force harvest in case the file is old and we're about to skip it
		logp.Info("Detected rename of a previously harvested file: %s -> %s", previous, filePath)

		lastState, _ := r.GetFileState(previous)
		lastState.Source = &filePath
		r.Persist <- lastState
		return lastState.Offset, true
	}

	if isFound {
		logp.Info("Not resuming rotated file: %s", filePath)
	}

	logp.Info("prospector", "New file. Start reading from the beginning: %s", filePath)

	// New file so just start from the beginning
	return 0, false
}
Example #4
0
func (p *ProspectorLog) Init() {
	logp.Debug("prospector", "exclude_files: %s", p.config.ExcludeFiles)

	logp.Info("Load previous states from registry into memory")
	fileStates := p.Prospector.states.GetStates()

	// Make sure all states are set as finished
	for _, state := range fileStates {

		// Check if state source belongs to this prospector. If yes, update the state.
		if p.hasFile(state.Source) {
			// Set all states again to infinity TTL to make sure only removed if config still same
			// clean_inactive / clean_removed could have been changed between restarts
			state.TTL = -1

			// Update prospector states and send new states to registry
			err := p.Prospector.updateState(input.NewEvent(state))
			if err != nil {
				logp.Err("Problem putting initial state: %+v", err)
			}
		} else {
			// Only update internal state, do not report it to registry
			// Having all states could be useful in case later a file is moved into this prospector
			// TODO: Think about if this is expected or unexpected
			p.Prospector.states.Update(state)
		}
	}

	p.lastClean = time.Now()

	logp.Info("Previous states loaded: %v", p.Prospector.states.Count())
}
Example #5
0
// Stop is called on exit before Cleanup
// why isn't the flow Cleanup and then Stop?
func (ub *Unifiedbeat) Stop() {
	startStopping := time.Now()
	logp.Info("Stop: is spooling and publishing running? '%v'", ub.isSpooling)
	if ub.isSpooling {
		ub.isSpooling = false
		logp.Info("Stop: waiting %v for spool/publish to shutdown.", ub.spoolTimeout)

		// lame, but might work
		time.Sleep(ub.spoolTimeout)

		// // tell "U2SpoolAndPublish" to shutdown:
		// quit <- true
		// // block/wait for "U2SpoolAndPublish" to close the quit channel:
		// <-quit

		err := ub.registrar.WriteRegistry()
		if err != nil {
			logp.Info("Stop: failed to update registry file; error: %v", err)
		} else {
			logp.Info("Stop: successfully updated registry file.")
		}
	}
	elapsed := time.Since(startStopping)
	logp.Info("Stop: done after waiting %v.", elapsed)
}
Example #6
0
func (r *Registrar) Run() {
	logp.Info("Starting Registrar")

	// Writes registry on shutdown
	defer r.writeRegistry()

	for {
		select {
		case <-r.done:
			logp.Info("Ending Registrar")
			return
		// Treats new log files to persist with higher priority then new events
		case state := <-r.Persist:
			r.State[*state.Source] = state
			logp.Debug("prospector", "Registrar will re-save state for %s", *state.Source)
		case events := <-r.Channel:
			r.processEvents(events)
		}

		if e := r.writeRegistry(); e != nil {
			// REVU: but we should panic, or something, right?
			logp.Err("Writing of registry returned error: %v. Continuing..", e)
		}
	}
}
Example #7
0
func newCloudMetadata(c common.Config) (processors.Processor, error) {
	config := struct {
		MetadataHostAndPort string        `config:"host"`    // Specifies the host and port of the metadata service (for testing purposes only).
		Timeout             time.Duration `config:"timeout"` // Amount of time to wait for responses from the metadata services.
	}{
		MetadataHostAndPort: metadataHost,
		Timeout:             3 * time.Second,
	}
	err := c.Unpack(&config)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unpack add_cloud_metadata config")
	}

	var (
		doURL  = "http://" + config.MetadataHostAndPort + doMetadataURI
		ec2URL = "http://" + config.MetadataHostAndPort + ec2InstanceIdentityURI
		gceURL = "http://" + config.MetadataHostAndPort + gceMetadataURI
	)

	result := fetchMetadata(doURL, ec2URL, gceURL, config.Timeout)
	if result == nil {
		logp.Info("add_cloud_metadata: hosting provider type not detected.")
		return addCloudMetadata{}, nil
	}

	logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v",
		result.provider, result.metadata.String())

	return addCloudMetadata{metadata: result.metadata}, nil
}
Example #8
0
// loadTemplate checks if the index mapping template should be loaded
// In case the template is not already loaded or overwritting is enabled, the
// template is written to index
func (out *elasticsearchOutput) loadTemplate(config Template, client *Client) error {
	out.templateMutex.Lock()
	defer out.templateMutex.Unlock()

	logp.Info("Trying to load template for client: %s", client)

	// Check if template already exist or should be overwritten
	exists := client.CheckTemplate(config.Name)
	if !exists || config.Overwrite {

		if config.Overwrite {
			logp.Info("Existing template will be overwritten, as overwrite is enabled.")
		}

		reader := bytes.NewReader(out.templateContents)
		err := client.LoadTemplate(config.Name, reader)
		if err != nil {
			return fmt.Errorf("Could not load template: %v", err)
		}
	} else {
		logp.Info("Template already exists and will not be overwritten.")
	}

	return nil
}
Example #9
0
func (d *Dockerbeat) Config(b *beat.Beat) error {

	// Requires Docker 1.5 minimum
	d.minimalDockerVersion = SoftwareVersion{major: 1, minor: 5}

	err := cfgfile.Read(&d.TbConfig, "")
	if err != nil {
		logp.Err("Error reading configuration file: %v", err)
		return err
	}

	//init the period
	if d.TbConfig.Input.Period != nil {
		d.period = time.Duration(*d.TbConfig.Input.Period) * time.Second
	} else {
		d.period = 1 * time.Second
	}
	//init the socket
	if d.TbConfig.Input.Socket != nil {
		d.socket = *d.TbConfig.Input.Socket
	} else {
		d.socket = "unix:///var/run/docker.sock" // default docker socket location
	}

	logp.Info("dockerbeat", "Init dockerbeat")
	logp.Info("dockerbeat", "Follow docker socket %q\n", d.socket)
	logp.Info("dockerbeat", "Period %v\n", d.period)

	return nil
}
Example #10
0
func (r *Registrar) WriteRegistry() error {
	r.Lock()
	defer r.Unlock()
	// can't truncate a file that does not exist:
	_, err := os.Stat(r.registryFile)
	if os.IsExist(err) {
		err := os.Truncate(r.registryFile, 0)
		if err != nil {
			logp.Info("WriteRegistry: os.Truncate: err=%v\n", err)
			return err
		}
	}
	// if "json.Marshal" or "ioutil.WriteFile" fail then most likely
	// unifiedbeat does not have access to the registry file
	jsonState, err := json.Marshal(r.State)
	if err != nil {
		logp.Info("WriteRegistry: json.Marshal: err=%v\n", err)
		return err
	}
	// https://golang.org/pkg/io/ioutil/#WriteFile
	// If the file does not exist, WriteFile creates it with
	// permissions 0644; otherwise it is truncated.
	err = ioutil.WriteFile(r.registryFile, jsonState, 0644)
	if err != nil {
		logp.Info("WriteRegistry: ioutil.WriteFile: err=%v\n", err)
		return err
	}

	return nil
}
Example #11
0
// Run calls the beater Setup and Run methods. In case of errors
// during the setup phase, it exits the process.
func (b *Beat) Run() error {

	// Setup beater object
	err := b.BT.Setup(b)
	if err != nil {
		return fmt.Errorf("setup returned an error: %v", err)
	}
	b.setState(SetupState)

	// Up to here was the initialization, now about running
	if cfgfile.IsTestConfig() {
		logp.Info("Testing configuration file")
		// all good, exit
		return nil
	}
	service.BeforeRun()

	// Callback is called if the processes is asked to stop.
	// This needs to be called before the main loop is started so that
	// it can register the signals that stop or query (on Windows) the loop.
	service.HandleSignals(b.Exit)

	logp.Info("%s sucessfully setup. Start running.", b.Name)

	b.setState(RunState)
	// Run beater specific stuff
	err = b.BT.Run(b)
	if err != nil {
		logp.Critical("Running the beat returned an error: %v", err)
	}

	return err
}
Example #12
0
func (c *Crawler) Start(prospectorConfigs []config.ProspectorConfig, eventChan chan *input.FileEvent) error {

	if len(prospectorConfigs) == 0 {
		return fmt.Errorf("No prospectors defined. You must have at least one prospector defined in the config file.")
	}

	logp.Info("Loading Prospectors: %v", len(prospectorConfigs))

	// Prospect the globs/paths given on the command line and launch harvesters
	for _, prospectorConfig := range prospectorConfigs {

		logp.Debug("prospector", "File Configs: %v", prospectorConfig.Paths)

		prospector, err := NewProspector(prospectorConfig, c.Registrar, eventChan)
		if err != nil {
			return fmt.Errorf("Error in initing prospector: %s", err)
		}
		c.prospectors = append(c.prospectors, prospector)
	}

	logp.Info("Loading Prospectors completed. Number of prospectors: %v", len(c.prospectors))

	c.wg = sync.WaitGroup{}
	for _, prospector := range c.prospectors {
		c.wg.Add(1)
		go prospector.Run(&c.wg)
	}

	logp.Info("All prospectors are initialised and running with %d states to persist", len(c.Registrar.State))

	return nil
}
Example #13
0
func (out *fileOutput) init(config config) error {
	out.rotator.Path = config.Path
	out.rotator.Name = config.Filename
	if out.rotator.Name == "" {
		out.rotator.Name = config.Index
	}
	logp.Info("File output path set to: %v", out.rotator.Path)
	logp.Info("File output base filename set to: %v", out.rotator.Name)

	rotateeverybytes := uint64(config.RotateEveryKb) * 1024
	logp.Info("Rotate every bytes set to: %v", rotateeverybytes)
	out.rotator.RotateEveryBytes = &rotateeverybytes

	keepfiles := config.NumberOfFiles
	logp.Info("Number of files set to: %v", keepfiles)
	out.rotator.KeepFiles = &keepfiles

	err := out.rotator.CreateDirectory()
	if err != nil {
		return err
	}

	err = out.rotator.CheckIfConfigSane()
	if err != nil {
		return err
	}

	return nil
}
Example #14
0
func (p *syncLogPublisher) Start() {
	p.wg.Add(1)
	go func() {
		defer p.wg.Done()

		logp.Info("Start sending events to output")

		for {
			var events []*input.FileEvent
			select {
			case <-p.done:
				return
			case events = <-p.in:
			}

			pubEvents := make([]common.MapStr, 0, len(events))
			for _, event := range events {
				pubEvents = append(pubEvents, event.ToMapStr())
			}

			p.client.PublishEvents(pubEvents, publisher.Sync, publisher.Guaranteed)
			logp.Info("Events sent: %d", len(events))

			// Tell the registrar that we've successfully sent these events
			select {
			case <-p.done:
				return
			case p.out <- events:
			}
		}
	}()
}
Example #15
0
// Starts the given module
func (m *Module) Start(b *beat.Beat) error {

	defer logp.Recover(fmt.Sprintf("Module %s paniced and stopped running.", m.name))

	if !m.Config.Enabled {
		logp.Debug("helper", "Not starting module %s with metricsets %s as not enabled.", m.name, m.getMetricSetsList())
		return nil
	}

	logp.Info("Setup moduler: %s", m.name)
	err := m.moduler.Setup(m)
	if err != nil {
		return fmt.Errorf("Error setting up module: %s. Not starting metricsets for this module.", err)
	}

	err = m.loadMetricsets()
	if err != nil {
		return fmt.Errorf("Error loading metricsets: %s", err)
	}

	// Setup period
	period, err := time.ParseDuration(m.Config.Period)
	if err != nil {
		return fmt.Errorf("Error in parsing period of module %s: %v", m.name, err)
	}

	// If no period set, set default
	if period == 0 {
		logp.Info("Setting default period for module %s as not set.", m.name)
		period = 1 * time.Second
	}

	var timeout time.Duration

	if m.Config.Timeout != "" {
		// Setup timeout
		timeout, err := time.ParseDuration(m.Config.Timeout)
		if err != nil {
			return fmt.Errorf("Error in parsing timeout of module %s: %v", m.name, err)
		}

		// If no timeout set, set to period as default
		if timeout == 0 {
			logp.Info("Setting default timeout for module %s as not set.", m.name)
			timeout = period
		}
	} else {
		timeout = period
	}
	m.Timeout = timeout

	logp.Info("Start Module %s with metricsets [%s] and period %v", m.name, m.getMetricSetsList(), period)

	m.setupMetricSets()

	m.events = b.Publisher.Connect()
	go m.Run(period, b)

	return nil
}
Example #16
0
func (r *Registrar) fetchState(filePath string, fileInfo os.FileInfo) (int64, bool) {

	// Check if there is a state for this file
	lastState, isFound := r.GetFileState(filePath)

	// Use os info to compare files as old fileInfo not necessary available
	if isFound && lastState.FileStateOS.IsSame(GetOSFileState(&fileInfo)) {

		logp.Debug("registar", "Same file as before found. Fetch the state.")
		// We're resuming - throw the last state back downstream so we resave it
		// And return the offset - also force harvest in case the file is old and we're about to skip it
		return lastState.Offset, true
	}

	if previous, err := r.getPreviousFile(filePath, fileInfo); err == nil {
		// File has rotated between shutdown and startup
		// We return last state downstream, with a modified event source with the new file name
		// And return the offset - also force harvest in case the file is old and we're about to skip it
		logp.Info("Detected rename of a previously harvested file: %s -> %s", previous, filePath)

		lastState, _ := r.GetFileState(previous)
		return lastState.Offset, true
	}

	if isFound {
		logp.Info("Not resuming rotated file: %s", filePath)
	}

	// New file so just start from an automatic position
	return 0, false
}
Example #17
0
func (r *Registrar) Run() {
	logp.Info("Starting Registrar")
	// Writes registry on shutdown
	defer func() {
		r.writeRegistry()
		r.wg.Done()
	}()

	for {
		select {
		case <-r.done:
			logp.Info("Ending Registrar")
			return
		case events := <-r.Channel:
			r.processEventStates(events)
		}

		beforeCount := r.states.Count()
		cleanedStates := r.states.Cleanup()
		logp.Debug("registrar", "Registrar states cleaned up. Before: %d , After: %d", beforeCount, beforeCount-cleanedStates)
		statesCleanup.Add(int64(cleanedStates))
		if err := r.writeRegistry(); err != nil {
			logp.Err("Writing of registry returned error: %v. Continuing...", err)
		}

	}
}
Example #18
0
// loadTemplate checks if the index mapping template should be loaded
// In case the template is not already loaded or overwritting is enabled, the
// template is written to index
func (out *elasticsearchOutput) loadTemplate(config Template, client *Client) error {
	out.templateMutex.Lock()
	defer out.templateMutex.Unlock()

	logp.Info("Trying to load template for client: %s", client.Connection.URL)

	// Check if template already exist or should be overwritten
	exists := client.CheckTemplate(config.Name)
	if !exists || config.Overwrite {

		if config.Overwrite {
			logp.Info("Existing template will be overwritten, as overwrite is enabled.")
		}

		template := out.template
		if config.Versions.Es2x.Enabled && strings.HasPrefix(client.Connection.version, "2.") {
			logp.Info("Detected Elasticsearch 2.x. Automatically selecting the 2.x version of the template")
			template = out.template2x
		}

		err := client.LoadTemplate(config.Name, template)
		if err != nil {
			return fmt.Errorf("Could not load template: %v", err)
		}
	} else {
		logp.Info("Template already exists and will not be overwritten.")
	}

	return nil
}
Example #19
0
// bulkCollectPublishFails checks per item errors returning all events
// to be tried again due to error code returned for that items. If indexing an
// event failed due to some error in the event itself (e.g. does not respect mapping),
// the event will be dropped.
func bulkCollectPublishFails(
	res *BulkResult,
	events []common.MapStr,
) []common.MapStr {
	failed := events[:0]
	for i, rawItem := range res.Items {
		status, msg, err := itemStatus(rawItem)
		if err != nil {
			logp.Info("Failed to parse bulk reponse for item (%i): %v", i, err)
			// add index if response parse error as we can not determine success/fail
			failed = append(failed, events[i])
			continue
		}

		if status < 300 {
			continue // ok value
		}

		if status < 500 && status != 429 {
			// hard failure, don't collect
			logp.Warn("Can not index event (status=%v): %v", status, msg)
			continue
		}

		debug("Failed to insert data(%v): %v", i, events[i])
		logp.Info("Bulk item insert failed (i=%v, status=%v): %v", i, status, msg)
		failed = append(failed, events[i])
	}
	return failed
}
Example #20
0
// dumpMetrics is used to log metrics on shutdown.
func dumpMetrics() {
	logp.Info("Dumping runtime metrics...")
	expvar.Do(func(kv expvar.KeyValue) {
		if kv.Key != "memstats" {
			logp.Info("%s=%s", kv.Key, kv.Value.String())
		}
	})
}
Example #21
0
// publish is used to publish events using the configured protocol client.
// It provides general error handling and back off support used on failed
// send attempts. To be used by PublishEvent and PublishEvents.
// The send callback will try to progress sending traffic and returns kind of
// progress made in ok or resetFail. If ok is set to true, send finished
// processing events. If ok is false but resetFail is set, send was partially
// successful. If send was partially successful, the fail counter is reset thus up
// to maxAttempts send attempts without any progress might be executed.
func (s *SingleConnectionMode) publish(
	signaler outputs.Signaler,
	opts outputs.Options,
	send func() (ok bool, resetFail bool),
) error {
	fails := 0
	var backoffCount uint
	var err error

	guaranteed := opts.Guaranteed || s.maxAttempts == 0
	for !s.closed && (guaranteed || fails < s.maxAttempts) {

		ok := false
		resetFail := false

		if err := s.connect(); err != nil {
			logp.Info("Connecting error publishing events (retrying): %s", err)
			goto sendFail
		}

		ok, resetFail = send()
		if !ok {
			goto sendFail
		}

		debug("send completed")
		outputs.SignalCompleted(signaler)
		return nil

	sendFail:
		fails++
		if resetFail {
			debug("reset fails")
			fails = 0
		}

		if !guaranteed && (s.maxAttempts > 0 && fails == s.maxAttempts) {
			// max number of attempts reached
			debug("max number of attempts reached")
			break
		}

		logp.Info("send fail")
		backoff := time.Duration(int64(s.waitRetry) * (1 << backoffCount))
		if backoff > s.maxWaitRetry {
			backoff = s.maxWaitRetry
		} else {
			backoffCount++
		}
		logp.Info("backoff retry: %v", backoff)
		time.Sleep(backoff)
	}

	debug("messages dropped")
	messagesDropped.Add(1)
	outputs.SignalFailed(signaler, err)
	return nil
}
Example #22
0
// Stop stops the spooler. Flushes events before stopping
func (s *Spooler) Stop() {
	logp.Info("Stopping spooler")
	close(s.exit)

	// Flush again before exiting spooler and closes channel
	logp.Info("Spooler last flush spooler")
	s.flush()
	close(s.Channel)
}
Example #23
0
// Cleanup performs clean-up after Run completes.
func (bt *Metricbeat) Cleanup(b *beat.Beat) error {
	logp.Info("Dumping runtime metrics...")
	expvar.Do(func(kv expvar.KeyValue) {
		if kv.Key != "memstats" {
			logp.Info("%s=%s", kv.Key, kv.Value.String())
		}
	})
	return nil
}
Example #24
0
func (proc *ProcessesWatcher) Init(config ProcsConfig) error {

	proc.proc_prefix = ""
	proc.PortProcMap = make(map[uint16]PortProcMapping)
	proc.LastMapUpdate = time.Now()

	proc.ReadFromProc = config.Enabled
	if proc.ReadFromProc {
		if runtime.GOOS != "linux" {
			proc.ReadFromProc = false
			logp.Info("Disabled /proc/ reading because not on linux")
		} else {
			logp.Info("Process matching enabled")
		}
	} else {
		logp.Info("Process matching disabled")
	}

	if config.Max_proc_read_freq == 0 {
		proc.MaxReadFreq = 10 * time.Millisecond
	} else {
		proc.MaxReadFreq = config.Max_proc_read_freq
	}

	if config.Refresh_pids_freq == 0 {
		proc.RefreshPidsFreq = 1 * time.Second
	} else {
		proc.RefreshPidsFreq = config.Refresh_pids_freq
	}

	// Read the local IP addresses
	var err error
	proc.LocalAddrs, err = common.LocalIpAddrs()
	if err != nil {
		logp.Err("Error getting local IP addresses: %s", err)
		proc.LocalAddrs = []net.IP{}
	}

	if proc.ReadFromProc {
		for _, procConfig := range config.Monitored {

			grepper := procConfig.Cmdline_grep
			if len(grepper) == 0 {
				grepper = procConfig.Process
			}

			p, err := NewProcess(proc, procConfig.Process, grepper, time.Tick(proc.RefreshPidsFreq))
			if err != nil {
				logp.Err("NewProcess: %s", err)
			} else {
				proc.Processes = append(proc.Processes, p)
			}
		}
	}

	return nil
}
Example #25
0
func (bt *Kafkabeat) Config(b *beat.Beat) error {

	// Load beater beatConfig
	logp.Info("Configuring Kafkabeat...")
	var err error
	err = cfgfile.Read(&bt.beatConfig, "")
	if err != nil {
		return fmt.Errorf("Error reading config file: %v", err)
	}

	bt.zookeepers = bt.beatConfig.Kafkabeat.Zookeepers
	if bt.zookeepers == nil || len(bt.zookeepers) == 0 {
		return KafkabeatError{"Atleast one zookeeper must be defined"}
	}
	zClient, err = kazoo.NewKazoo(bt.zookeepers, nil)
	if err != nil {
		logp.Err("Unable to connect to Zookeeper")
		return err
	}
	bt.brokers, err = zClient.BrokerList()
	if err != nil {
		logp.Err("Error identifying brokers from zookeeper")
		return err
	}
	if bt.brokers == nil || len(bt.brokers) == 0 {
		return KafkabeatError{"Unable to identify active brokers"}
	}
	logp.Info("Brokers: %v", bt.brokers)
	client, err = sarama.NewClient(bt.brokers, sarama.NewConfig())

	// Setting default period if not set
	if bt.beatConfig.Kafkabeat.Period == "" {
		bt.beatConfig.Kafkabeat.Period = "1s"
	}

	bt.period, err = time.ParseDuration(bt.beatConfig.Kafkabeat.Period)
	if err != nil {
		return err
	}
	bt.topics = bt.beatConfig.Kafkabeat.Topics
	bt.create_topic_docs = true
	if bt.topics == nil || len(bt.topics) == 0 {
		bt.create_topic_docs = bt.topics == nil
		bt.topics, err = client.Topics()
	}
	if err != nil {
		return err
	}
	logp.Info("Monitoring topics: %v", bt.topics)
	bt.groups = bt.beatConfig.Kafkabeat.Groups
	if bt.groups == nil {
		bt.groups, err = getGroups()
	}
	logp.Info("Monitoring groups %v", bt.groups)
	return err
}
Example #26
0
func (c *Crawler) Stop() {
	logp.Info("Stopping Crawler")

	logp.Info("Stopping %v prospectors", len(c.prospectors))
	for _, prospector := range c.prospectors {
		prospector.Stop()
	}
	c.wg.Wait()
	logp.Info("Crawler stopped")
}
Example #27
0
func (ub *Unifiedbeat) Cleanup(b *beat.Beat) error {
	logp.Info("Cleanup: is spooling and publishing running? '%v'", ub.isSpooling)
	// see "beat/geoip2.go":
	if GeoIp2Reader != nil {
		GeoIp2Reader.Close()
		logp.Info("Cleanup: closed GeoIp2Reader.")
	}
	logp.Info("Cleanup: done.")
	return nil
}
Example #28
0
func (c *Crawler) Stop() {
	logp.Info("Stopping Crawler")

	logp.Info("Stopping %v prospectors", len(c.prospectors))
	for _, prospector := range c.prospectors {
		// Stop prospectors in parallel
		c.wg.Add(1)
		go prospector.Stop(&c.wg)
	}
	c.wg.Wait()
	logp.Info("Crawler stopped")
}
Example #29
0
// publish is used to publish events using the configured protocol client.
// It provides general error handling and back off support used on failed
// send attempts. To be used by PublishEvent and PublishEvents.
// The send callback will try to progress sending traffic and returns kind of
// progress made in ok or resetFail. If ok is set to true, send finished
// processing events. If ok is false but resetFail is set, send was partially
// successful. If send was partially successful, the fail counter is reset thus up
// to maxAttempts send attempts without any progress might be executed.
func (s *Mode) publish(
	signaler op.Signaler,
	opts outputs.Options,
	send func() (ok bool, resetFail bool),
) error {
	fails := 0
	var err error

	guaranteed := opts.Guaranteed || s.maxAttempts == 0
	for !s.closed && (guaranteed || fails < s.maxAttempts) {

		ok := false
		resetFail := false

		if err := s.connect(); err != nil {
			logp.Info("Connecting error publishing events (retrying): %s", err)
			goto sendFail
		}

		ok, resetFail = send()
		if !ok {
			goto sendFail
		}

		debugf("send completed")
		s.backoff.Reset()
		op.SigCompleted(signaler)
		return nil

	sendFail:
		logp.Info("send fail")

		fails++
		if resetFail {
			debugf("reset fails")
			s.backoff.Reset()
			fails = 0
		}
		s.backoff.Wait()

		if !guaranteed && (s.maxAttempts > 0 && fails == s.maxAttempts) {
			// max number of attempts reached
			debugf("max number of attempts reached")
			break
		}
	}

	debugf("messages dropped")
	mode.Dropped(1)
	op.SigFailed(signaler, err)
	return nil
}
Example #30
0
File: redis.go Project: jarpy/beats
func (out *redisOutput) Init(config *redisConfig, topology_expire int) error {

	logp.Warn("Redis Output is deprecated. Please use the Redis Output Plugin from Logstash instead.")

	out.Hostname = fmt.Sprintf("%s:%d", config.Host, config.Port)
	out.Password = config.Password
	out.Index = config.Index
	out.Db = config.Db
	out.DbTopology = config.DbTopology

	out.Timeout = 5 * time.Second
	if config.Timeout > 0 {
		out.Timeout = time.Duration(config.Timeout) * time.Second
	}

	out.ReconnectInterval = time.Duration(1) * time.Second
	if config.ReconnectInterval >= 0 {
		out.ReconnectInterval = time.Duration(config.ReconnectInterval) * time.Second
	}
	logp.Info("Reconnect Interval set to: %v", out.ReconnectInterval)

	expSec := 15
	if topology_expire != 0 {
		expSec = topology_expire
	}
	out.TopologyExpire = time.Duration(expSec) * time.Second

	switch config.DataType {
	case "", "list":
		out.DataType = RedisListType
	case "channel":
		out.DataType = RedisChannelType
	default:
		return errors.New("Bad Redis data type")
	}

	logp.Info("[RedisOutput] Using Redis server %s", out.Hostname)
	if out.Password != "" {
		logp.Info("[RedisOutput] Using password to connect to Redis")
	}
	logp.Info("[RedisOutput] Redis connection timeout %s", out.Timeout)
	logp.Info("[RedisOutput] Redis reconnect interval %s", out.ReconnectInterval)
	logp.Info("[RedisOutput] Using index pattern %s", out.Index)
	logp.Info("[RedisOutput] Topology expires after %s", out.TopologyExpire)
	logp.Info("[RedisOutput] Using db %d for storing events", out.Db)
	logp.Info("[RedisOutput] Using db %d for storing topology", out.DbTopology)
	logp.Info("[RedisOutput] Using %d data type", out.DataType)

	out.Reconnect()

	return nil
}