Esempio n. 1
0
// Main Logstreamer Input runner. This runner kicks off all the other
// logstream inputs, and handles rescanning for updates to the filesystem that
// might affect file visibility for the logstream inputs.
func (li *LogstreamerInput) Run(ir p.InputRunner, h p.PluginHelper) (err error) {
	var (
		ok         bool
		errs       *ls.MultipleError
		newstreams []string
	)

	// Kick off all the current logstreams we know of
	i := 0
	for _, logstream := range li.plugins {
		i++
		li.startLogstreamInput(logstream, i, ir, h)
	}

	ok = true
	rescan := time.Tick(li.rescanInterval)
	// Our main rescan loop that handles shutting down
	for ok {
		select {
		case <-li.stopChan:
			ok = false
			returnChans := make([]chan bool, len(li.stopLogstreamChans))
			// Send out all the stop signals
			for i, ch := range li.stopLogstreamChans {
				ret := make(chan bool)
				ch <- ret
				returnChans[i] = ret
			}

			// Wait for all the stops
			for _, ch := range returnChans {
				<-ch
			}

			// Close our own stopChan to indicate we shut down
			close(li.stopChan)
		case <-rescan:
			li.logstreamSetLock.Lock()
			newstreams, errs = li.logstreamSet.ScanForLogstreams()
			if errs.IsError() {
				ir.LogError(errs)
			}
			for _, name := range newstreams {
				stream, ok := li.logstreamSet.GetLogstream(name)
				if !ok {
					ir.LogError(fmt.Errorf("Found new logstream: %s, but couldn't fetch it.",
						name))
					continue
				}

				lsi := NewLogstreamInput(stream, name, li.hostName)
				li.plugins[name] = lsi
				i++
				li.startLogstreamInput(lsi, i, ir, h)
			}
			li.logstreamSetLock.Unlock()
		}
	}
	return nil
}
Esempio n. 2
0
func (li *LogstreamerInput) Init(config interface{}) (err error) {
	var (
		errs    *ls.MultipleError
		oldest  time.Duration
		plugins []string
	)
	conf := config.(*LogstreamerInputConfig)

	// Setup the journal dir
	if err = os.MkdirAll(conf.JournalDirectory, 0744); err != nil {
		return err
	}

	if len(conf.FileMatch) > 0 && conf.FileMatch[len(conf.FileMatch)-1:] != "$" {
		conf.FileMatch += "$"
	}

	li.decoderName = conf.Decoder
	li.parser = conf.ParserType
	li.delimiter = conf.Delimiter
	li.delimiterLocation = conf.DelimiterLocation
	li.plugins = make(map[string]*LogstreamInput)

	// Setup the rescan interval
	if li.rescanInterval, err = time.ParseDuration(conf.RescanInterval); err != nil {
		return
	}

	// Parse the oldest duration
	if oldest, err = time.ParseDuration(conf.OldestDuration); err != nil {
		return
	}

	// If no differentiator is present than we use the plugin
	if len(conf.Differentiator) == 0 {
		conf.Differentiator = []string{li.pluginName}
	}

	for name, submap := range conf.Translation {
		if len(submap) == 1 {
			if _, ok := submap["missing"]; !ok {
				err = fmt.Errorf("A translation map with one entry ('%s') must be "+
					"specifying a 'missing' key.", name)
				return
			}
		}
	}

	// Create the main sort pattern
	sp := &ls.SortPattern{
		FileMatch:      conf.FileMatch,
		Translation:    conf.Translation,
		Priority:       conf.Priority,
		Differentiator: conf.Differentiator,
	}

	// Create the main logstream set
	li.logstreamSetLock.Lock()
	defer li.logstreamSetLock.Unlock()
	li.logstreamSet, err = ls.NewLogstreamSet(sp, oldest, conf.LogDirectory,
		conf.JournalDirectory)
	if err != nil {
		return
	}

	// Initial scan for logstreams
	plugins, errs = li.logstreamSet.ScanForLogstreams()
	if errs.IsError() {
		return errs
	}

	// Verify we can make a parser
	if _, _, err = CreateParser(li.parser, li.delimiter,
		li.delimiterLocation, li.decoderName); err != nil {
		return
	}

	// Declare our hostname
	if conf.Hostname == "" {
		li.hostName, err = os.Hostname()
		if err != nil {
			return
		}
	} else {
		li.hostName = conf.Hostname
	}

	// Create all our initial logstream plugins for the logstreams found
	for _, name := range plugins {
		stream, ok := li.logstreamSet.GetLogstream(name)
		if !ok {
			continue
		}
		stParser, parserFunc, _ := CreateParser(li.parser, li.delimiter,
			li.delimiterLocation, li.decoderName)
		li.plugins[name] = NewLogstreamInput(stream, stParser, parserFunc,
			name, li.hostName)
	}
	li.stopLogstreamChans = make([]chan chan bool, 0, len(plugins))
	li.stopChan = make(chan bool)
	return
}
Esempio n. 3
0
// Main Logstreamer Input runner
// This runner kicks off all the other logstream inputs, and handles rescanning for
// updates to the filesystem that might affect file visibility for the logstream
// inputs
func (li *LogstreamerInput) Run(ir p.InputRunner, h p.PluginHelper) (err error) {
	var (
		ok         bool
		dRunner    p.DecoderRunner
		errs       *ls.MultipleError
		newstreams []string
	)

	// Setup the decoder runner that will be used
	if li.decoderName != "" {
		if dRunner, ok = h.DecoderRunner(li.decoderName, fmt.Sprintf("%s-%s",
			li.pluginName, li.decoderName)); !ok {

			return fmt.Errorf("Decoder not found: %s", li.decoderName)
		}
	}

	// Kick off all the current logstreams we know of
	for _, logstream := range li.plugins {
		stop := make(chan chan bool, 1)
		go logstream.Run(ir, h, stop, dRunner)
		li.stopLogstreamChans = append(li.stopLogstreamChans, stop)
	}

	ok = true
	rescan := time.Tick(li.rescanInterval)
	// Our main rescan loop that handles shutting down
	for ok {
		select {
		case <-li.stopChan:
			ok = false
			returnChans := make([]chan bool, len(li.stopLogstreamChans))
			// Send out all the stop signals
			for i, ch := range li.stopLogstreamChans {
				ret := make(chan bool)
				ch <- ret
				returnChans[i] = ret
			}

			// Wait for all the stops
			for _, ch := range returnChans {
				<-ch
			}

			// Close our own stopChan to indicate we shut down
			close(li.stopChan)
		case <-rescan:
			li.logstreamSetLock.Lock()
			newstreams, errs = li.logstreamSet.ScanForLogstreams()
			if errs.IsError() {
				ir.LogError(errs)
			}
			for _, name := range newstreams {
				stream, ok := li.logstreamSet.GetLogstream(name)
				if !ok {
					ir.LogError(fmt.Errorf("Found new logstream: %s, but couldn't fetch it.", name))
					continue
				}

				// Setup a new logstream input for this logstream and start it running
				stParser, parserFunc, _ := CreateParser(li.parser, li.delimiter,
					li.delimiterLocation, li.decoderName)

				lsi := NewLogstreamInput(stream, stParser, parserFunc, name,
					li.hostName)
				li.plugins[name] = lsi
				stop := make(chan chan bool, 1)
				go lsi.Run(ir, h, stop, dRunner)
				li.stopLogstreamChans = append(li.stopLogstreamChans, stop)
			}
			li.logstreamSetLock.Unlock()
		}
	}
	err = nil
	return
}