コード例 #1
0
ファイル: publish.go プロジェクト: venkateshdaram434/libbeat
func (publisher *PublisherType) publishEvents(events []common.MapStr) {
	var ignore []int // indices of events to be removed from events
	for i, event := range events {
		// validate some required field
		if err := filterEvent(event); err != nil {
			logp.Err("Publishing event failed: %v", err)
			ignore = append(ignore, i)
			continue
		}

		// update address and geo-ip information. Ignore event
		// if address is invalid or event is found to be a duplicate
		ok := updateEventAddresses(publisher, event)
		if !ok {
			ignore = append(ignore, i)
			continue
		}

		// add additional meta data
		event["shipper"] = publisher.name
		if len(publisher.tags) > 0 {
			event["tags"] = publisher.tags
		}

		if logp.IsDebug("publish") {
			PrintPublishEvent(event)
		}
	}

	// return if no event is left
	if len(ignore) == len(events) {
		return
	}

	// remove invalid events.
	// TODO: is order important? Removal can be turned into O(len(ignore)) by
	//       copying last element into idx and doing
	//       events=events[:len(events)-len(ignore)] afterwards
	// Alternatively filtering could be implemented like:
	//   https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
	for i := len(ignore) - 1; i >= 0; i-- {
		idx := ignore[i]
		events = append(events[:idx], events[idx+1:]...)
	}

	// get timestamp of first event for outputer
	ts := events[0]["timestamp"].(common.Time)

	// add transaction
	if !publisher.disabled {
		for _, out := range publisher.Output {
			err := out.BulkPublish(time.Time(ts), events)
			if err != nil {
				logp.Err("Fail to publish event type on output %s: %v", out, err)
			}
		}
	}
}
コード例 #2
0
ファイル: preprocess.go プロジェクト: rshriram/libbeat
func (p *preprocessor) onMessage(m message) {
	publisher := p.pub
	single := false
	events := m.events
	if m.event != nil {
		single = true
		events = []common.MapStr{m.event}
	}

	var ignore []int // indices of events to be removed from events

	debug("preprocessor")

	for i, event := range events {
		// validate some required field
		if err := filterEvent(event); err != nil {
			logp.Err("Publishing event failed: %v", err)
			ignore = append(ignore, i)
			continue
		}

		// update address and geo-ip information. Ignore event
		// if address is invalid or event is found to be a duplicate
		ok := updateEventAddresses(publisher, event)
		if !ok {
			ignore = append(ignore, i)
			continue
		}

		// add additional Beat meta data
		event["beat"] = common.MapStr{
			"name":     publisher.name,
			"hostname": publisher.hostname,
			"version":  publisher.version,
		}
		if len(publisher.tags) > 0 {
			event["tags"] = publisher.tags
		}

		if logp.IsDebug("publish") {
			PrintPublishEvent(event)
		}
	}

	// return if no event is left
	if len(ignore) == len(events) {
		debug("no event left, complete send")
		outputs.SignalCompleted(m.context.signal)
		return
	}

	// remove invalid events.
	// TODO: is order important? Removal can be turned into O(len(ignore)) by
	//       copying last element into idx and doing
	//       events=events[:len(events)-len(ignore)] afterwards
	// Alternatively filtering could be implemented like:
	//   https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
	for i := len(ignore) - 1; i >= 0; i-- {
		idx := ignore[i]
		debug("remove event[%v]", idx)
		events = append(events[:idx], events[idx+1:]...)
	}

	if publisher.disabled {
		debug("publisher disabled")
		outputs.SignalCompleted(m.context.signal)
		return
	}

	debug("preprocessor forward")
	if single {
		p.handler.onMessage(message{context: m.context, event: events[0]})
	} else {
		p.handler.onMessage(message{context: m.context, events: events})
	}
}
コード例 #3
0
ファイル: publish.go プロジェクト: ruflin/dfbeat
func (publisher *PublisherType) publishEvent(event common.MapStr) error {

	// the timestamp is mandatory
	ts, ok := event["timestamp"].(common.Time)
	if !ok {
		return errors.New("Missing 'timestamp' field from event.")
	}

	// the count is mandatory
	err := event.EnsureCountField()
	if err != nil {
		return err
	}

	// the type is mandatory
	_, ok = event["type"].(string)
	if !ok {
		return errors.New("Missing 'type' field from event.")
	}

	var src_server, dst_server string
	src, ok := event["src"].(*common.Endpoint)
	if ok {
		src_server = publisher.GetServerName(src.Ip)
		event["client_ip"] = src.Ip
		event["client_port"] = src.Port
		event["client_proc"] = src.Proc
		event["client_server"] = src_server
		delete(event, "src")
	}
	dst, ok := event["dst"].(*common.Endpoint)
	if ok {
		dst_server = publisher.GetServerName(dst.Ip)
		event["ip"] = dst.Ip
		event["port"] = dst.Port
		event["proc"] = dst.Proc
		event["server"] = dst_server
		delete(event, "dst")
	}

	if publisher.IgnoreOutgoing && dst_server != "" &&
		dst_server != publisher.name {
		// duplicated transaction -> ignore it
		logp.Debug("publish", "Ignore duplicated transaction on %s: %s -> %s", publisher.name, src_server, dst_server)
		return nil
	}

	event["shipper"] = publisher.name
	if len(publisher.tags) > 0 {
		event["tags"] = publisher.tags
	}

	if publisher.GeoLite != nil {
		real_ip, exists := event["real_ip"]
		if exists && len(real_ip.(string)) > 0 {
			loc := publisher.GeoLite.GetLocationByIP(real_ip.(string))
			if loc != nil && loc.Latitude != 0 && loc.Longitude != 0 {
				event["client_location"] = fmt.Sprintf("%f, %f", loc.Latitude, loc.Longitude)
			}
		} else {
			if len(src_server) == 0 && src != nil { // only for external IP addresses
				loc := publisher.GeoLite.GetLocationByIP(src.Ip)
				if loc != nil && loc.Latitude != 0 && loc.Longitude != 0 {
					event["client_location"] = fmt.Sprintf("%f, %f", loc.Latitude, loc.Longitude)
				}
			}
		}
	}

	if logp.IsDebug("publish") {
		PrintPublishEvent(event)
	}

	// add transaction
	has_error := false
	if !publisher.disabled {
		for i := 0; i < len(publisher.Output); i++ {
			err := publisher.Output[i].PublishEvent(time.Time(ts), event)
			if err != nil {
				logp.Err("Fail to publish event type on output %s: %v", publisher.Output[i], err)
				has_error = true
			}
		}
	}

	if has_error {
		return errors.New("Fail to publish event")
	}
	return nil
}
コード例 #4
0
ファイル: main.go プロジェクト: Esquive/packetbeat
func main() {

	// Use our own FlagSet, because some libraries pollute the global one
	var cmdLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)

	//	configfile := cmdLine.String("c", "packetbeat.dev.yml", "Configuration file")
	configfile := cmdLine.String("c", "/etc/packetbeat/packetbeat.yml", "Configuration file")
	file := cmdLine.String("I", "", "file")
	loop := cmdLine.Int("l", 1, "Loop file. 0 - loop forever")
	debugSelectorsStr := cmdLine.String("d", "", "Enable certain debug selectors")
	oneAtAtime := cmdLine.Bool("O", false, "Read packets one at a time (press Enter)")
	toStderr := cmdLine.Bool("e", false, "Output to stdout instead of syslog")
	topSpeed := cmdLine.Bool("t", false, "Read packets as fast as possible, without sleeping")
	publishDisabled := cmdLine.Bool("N", false, "Disable actual publishing for testing")
	verbose := cmdLine.Bool("v", false, "Log at INFO level")
	printVersion := cmdLine.Bool("version", false, "Print version and exit")
	memprofile := cmdLine.String("memprofile", "", "Write memory profile to this file")
	cpuprofile := cmdLine.String("cpuprofile", "", "Write cpu profile to file")
	dumpfile := cmdLine.String("dump", "", "Write all captured packets to this libpcap file.")
	testConfig := cmdLine.Bool("test", false, "Test configuration and exit.")

	cmdLine.Parse(os.Args[1:])

	sniff := new(sniffer.SnifferSetup)

	if *printVersion {
		fmt.Printf("Packetbeat version %s (%s)\n", Version, runtime.GOARCH)
		return
	}

	logLevel := logp.LOG_ERR
	if *verbose {
		logLevel = logp.LOG_INFO
	}

	debugSelectors := []string{}
	if len(*debugSelectorsStr) > 0 {
		debugSelectors = strings.Split(*debugSelectorsStr, ",")
		logLevel = logp.LOG_DEBUG
	}

	var err error

	filecontent, err := ioutil.ReadFile(*configfile)
	if err != nil {
		fmt.Printf("Fail to read %s: %s. Exiting.\n", *configfile, err)
		return
	}
	if err = yaml.Unmarshal(filecontent, &config.ConfigSingleton); err != nil {
		fmt.Printf("YAML config parsing failed on %s: %s. Exiting.\n", *configfile, err)
		return
	}

	if len(debugSelectors) == 0 {
		debugSelectors = config.ConfigSingleton.Logging.Selectors
	}
	logp.LogInit(logp.Priority(logLevel), "", !*toStderr, true, debugSelectors)

	if !logp.IsDebug("stdlog") {
		// disable standard logging by default
		log.SetOutput(ioutil.Discard)
	}

	// CLI flags over-riding config
	if *topSpeed {
		config.ConfigSingleton.Interfaces.TopSpeed = true
	}
	if len(*file) > 0 {
		config.ConfigSingleton.Interfaces.File = *file
	}
	config.ConfigSingleton.Interfaces.Loop = *loop
	config.ConfigSingleton.Interfaces.OneAtATime = *oneAtAtime
	if len(*dumpfile) > 0 {
		config.ConfigSingleton.Interfaces.Dumpfile = *dumpfile
	}

	logp.Debug("main", "Configuration %s", config.ConfigSingleton)
	logp.Debug("main", "Initializing output plugins")
	if err = publisher.Publisher.Init(*publishDisabled, config.ConfigSingleton.Output,
		config.ConfigSingleton.Shipper); err != nil {

		logp.Critical(err.Error())
		os.Exit(1)
	}

	if err = procs.ProcWatcher.Init(config.ConfigSingleton.Procs); err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	logp.Debug("main", "Initializing protocol plugins")
	for proto, plugin := range EnabledProtocolPlugins {

		logp.Debug("protos", proto.String())

		err = plugin.Init(false, publisher.Publisher.Queue)
		if err != nil {
			logp.Critical("Initializing plugin %s failed: %v", proto, err)
			os.Exit(1)
		}
		protos.Protos.Register(proto, plugin)
	}

	if err = tcp.TcpInit(); err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	over := make(chan bool)

	logp.Debug("main", "Initializing filters plugins")
	for filter, plugin := range EnabledFilterPlugins {
		filters.Filters.Register(filter, plugin)
	}
	filters_plugins, err :=
		LoadConfiguredFilters(config.ConfigSingleton.Filter)
	if err != nil {
		logp.Critical("Error loading filters plugins: %v", err)
		os.Exit(1)
	}
	logp.Debug("main", "Filters plugins order: %v", filters_plugins)
	var afterInputsQueue chan common.MapStr
	if len(filters_plugins) > 0 {
		runner := NewFilterRunner(publisher.Publisher.Queue, filters_plugins)
		go func() {
			err := runner.Run()
			if err != nil {
				logp.Critical("Filters runner failed: %v", err)
				// shutting doen
				sniff.Stop()
			}
		}()
		afterInputsQueue = runner.FiltersQueue
	} else {
		// short-circuit the runner
		afterInputsQueue = publisher.Publisher.Queue
	}

	logp.Debug("main", "Initializing sniffer")
	err = sniff.Init(false, afterInputsQueue)
	if err != nil {
		logp.Critical("Initializing sniffer failed: %v", err)
		os.Exit(1)
	}

	// This needs to be after the sniffer Init but before the sniffer Run.
	if err = droppriv.DropPrivileges(config.ConfigSingleton.RunOptions); err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	// Up to here was the initialization, now about running

	if *testConfig {
		// all good, exit with 0
		os.Exit(0)
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// run the sniffer in background
	go func() {
		err := sniff.Run()
		if err != nil {
			logp.Critical("Sniffer main loop failed: %v", err)
			os.Exit(1)
		}
		over <- true
	}()

	// On ^C or SIGTERM, gracefully stop the sniffer
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		<-sigc
		logp.Debug("signal", "Received sigterm/sigint, stopping")
		sniff.Stop()
	}()

	if !*toStderr {
		logp.Info("Startup successful, sending output only to syslog from now on")
		logp.SetToStderr(false)
	}

	logp.Debug("main", "Waiting for the sniffer to finish")

	// Wait for the goroutines to finish
	for _ = range over {
		if !sniff.IsAlive() {
			break
		}
	}

	logp.Debug("main", "Cleanup")

	if *memprofile != "" {
		// wait for all TCP streams to expire
		time.Sleep(tcp.TCP_STREAM_EXPIRY * 1.2)
		tcp.PrintTcpMap()
		runtime.GC()

		writeHeapProfile(*memprofile)

		debugMemStats()
	}
}