示例#1
0
func TestAggregateSum(t *testing.T) {

	log.SetLevel(log.ERROR)

	es := New("http://10.16.1.10:9200")

	size := int64(10)

	eventType := "netflow"

	query := NewEventQuery()
	query.AddFilter(TermQuery("event_type", eventType))

	aggType := "keyword"

	agg := "src_ip"

	sum := m{
		"sum": m{
			"sum": m{
				"field": "netflow.pkts",
			},
		},
	}

	if aggType == "keyword" {
		query.Aggs[agg] = map[string]interface{}{
			"terms": map[string]interface{}{
				"field": fmt.Sprintf("%s.%s", agg, es.keyword),
				"size":  size,
			},
		}
	} else {
		query.Aggs[agg] = map[string]interface{}{
			"terms": map[string]interface{}{
				"field": agg,
				"size":  size,
			},
		}
	}

	query.Aggs[agg].(map[string]interface{})["aggs"] = sum

	fmt.Println(ToJsonPretty(query))

	response, err := es.Search(query)
	if err != nil {
		t.Fatal(err)
	}

	fmt.Println(ToJsonPretty(response))

}
示例#2
0
func Main(args []string) {

	var end bool
	var oneshot bool
	var useBookmark bool
	var bookmarkPath string
	var verbose bool
	var dbfile string

	flagset := flag.NewFlagSet("sqliteimport", flag.ExitOnError)
	flagset.StringVarP(&dbfile, "database", "D", "", "Database filename")
	flagset.BoolVar(&end, "end", false, "Start at end of file")
	flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
	flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
	flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
	flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
	flagset.Parse(args)

	if verbose {
		log.SetLevel(log.DEBUG)
	}

	if dbfile == "" {
		log.Fatal("Database filename must be provided.")
	}

	if len(flagset.Args()) == 0 {
		log.Fatal("No input files provided.")
	} else if len(flagset.Args()) > 1 {
		log.Fatal("Only one input file allowed.")
	}
	inputFilename := flagset.Args()[0]

	// If useBookmark but no path, set a default.
	if useBookmark && bookmarkPath == "" {
		bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
	}

	reader, err := evereader.New(flagset.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	// Initialize bookmark.
	var bookmarker *evereader.Bookmarker = nil
	if useBookmark {
		bookmarker = &evereader.Bookmarker{
			Filename: bookmarkPath,
			Reader:   reader,
		}
		err := bookmarker.Init(end)
		if err != nil {
			log.Fatal(err)
		}
	} else if end {
		log.Info("Jumping to end of file.")
		err := reader.SkipToEnd()
		if err != nil {
			log.Fatal(err)
		}
	}

	db, err := sqlite.NewSqliteService(dbfile)
	if err != nil {
		log.Fatal(err)
	}
	if err := db.Migrate(); err != nil {
		log.Fatal(err)
	}

	indexer, err := sqlite.NewSqliteIndexer(db)
	if err != nil {
		log.Fatal(err)
	}

	count := uint64(0)
	lastStatTs := time.Now()
	lastStatCount := uint64(0)

	// Number of EOFs in last stat interval.
	eofs := uint64(0)

	tagsFilter := eve.TagsFilter{}

	for {
		eof := false
		event, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				eof = true
				eofs++
			} else {
				log.Fatal(err)
			}
		}

		if event != nil {

			tagsFilter.Filter(event)

			indexer.IndexRawEve(event)
			count++

			if useBookmark {
				bookmark := bookmarker.GetBookmark()
				bookmarker.WriteBookmark(bookmark)
			}
		}

		now := time.Now()

		if now.Sub(lastStatTs).Seconds() > 1 {
			log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
				count,
				count-lastStatCount,
				float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
				eofs)
			lastStatTs = now
			lastStatCount = count
			eofs = 0
			indexer.Flush()
		}

		if eof {
			if oneshot {
				break
			} else {
				indexer.Flush()
				time.Sleep(100 * time.Millisecond)
			}
		}

	}

	now := time.Now()

	log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
		count,
		count-lastStatCount,
		float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
		eofs)

	indexer.Flush()
}
示例#3
0
func Main(args []string) {

	var end bool
	var oneshot bool
	var useBookmark bool
	var bookmarkPath string
	var verbose bool

	flagset := flag.NewFlagSet("evereader", flag.ExitOnError)
	flagset.BoolVar(&end, "end", false, "Start at end of file")
	flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
	flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
	flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
	flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
	flagset.Parse(args[1:])

	if verbose {
		log.SetLevel(log.DEBUG)
	}

	if len(flagset.Args()) == 0 {
		log.Fatal("No input files provided.")
	} else if len(flagset.Args()) > 1 {
		log.Fatal("Only one input file allowed.")
	}
	inputFilename := flagset.Args()[0]

	// If useBookmark but no path, set a default.
	if useBookmark && bookmarkPath == "" {
		bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
	}

	encoder := json.NewEncoder(os.Stdout)

	reader, err := evereader.New(flagset.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	// Initialize bookmark.
	var bookmarker *evereader.Bookmarker = nil
	if useBookmark {
		bookmarker = &evereader.Bookmarker{
			Filename: bookmarkPath,
			Reader:   reader,
		}
		err := bookmarker.Init(end)
		if err != nil {
			log.Fatal(err)
		}
	} else if end {
		log.Info("Jumping to end of file.")
		err := reader.SkipToEnd()
		if err != nil {
			log.Fatal(err)
		}
	}

	for {
		eof := false
		event, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				if oneshot {
					break
				}
				eof = true
			} else {
				log.Fatal(err)
			}
		}

		if eof {
			time.Sleep(1 * time.Second)
		} else {
			encoder.Encode(event)

			if useBookmark {
				bookmark := bookmarker.GetBookmark()
				bookmarker.WriteBookmark(bookmark)
			}
		}
	}
}
示例#4
0
文件: server.go 项目: jasonish/evebox
func Main(args []string) {

	var err error

	log.Info("This is EveBox Server version %v (rev: %v)", core.BuildVersion, core.BuildRev)

	flagset := flag.NewFlagSet("server", flag.ExitOnError)

	flagset.StringVarP(&opts.ElasticSearchUri, "elasticsearch", "e", "", "Elastic Search URI (default: http://localhost:9200")
	flagset.StringVarP(&opts.ElasticSearchIndex, "index", "i", "", "Elastic Search Index (default: logstash)")
	flagset.StringVarP(&opts.Port, "port", "p", "5636", "Port to bind to")
	flagset.StringVarP(&opts.Host, "host", "", "0.0.0.0", "Host to bind to")
	flagset.StringVarP(&opts.DevServerUri, "dev", "", "", "Frontend development server URI")
	flagset.BoolVarP(&opts.Version, "version", "", false, "Show version")
	flagset.StringVarP(&opts.Config, "config", "c", "", "Configuration filename")
	flagset.BoolVarP(&opts.NoCheckCertificate, "no-check-certificate", "k", false, "Disable certificate check for Elastic Search")

	flagset.BoolVarP(&opts.Sqlite, "sqlite", "", false, "Use SQLite for the event store")

	flagset.Parse(args[0:])

	if opts.Version {
		VersionMain()
		return
	}

	log.SetLevel(log.DEBUG)

	// If no configuration was provided, see if evebox.yaml exists
	// in the current directory.
	if opts.Config == "" {
		_, err = os.Stat("./evebox.yaml")
		if err == nil {
			opts.Config = "./evebox.yaml"
		}
	}
	if opts.Config != "" {
		log.Printf("Loading configuration file %s.\n", opts.Config)
		conf, err = config.LoadConfig(opts.Config)
		if err != nil {
			log.Fatal(err)
		}
	}

	conf.ElasticSearchIndex = getElasticSearchIndex()
	log.Info("Using ElasticSearch Index %s.", conf.ElasticSearchIndex)

	appContext := server.AppContext{
		Config: conf,
	}
	elasticSearch := elasticsearch.New(getElasticSearchUrl())
	elasticSearch.SetEventIndex(conf.ElasticSearchIndex)
	elasticSearch.InitKeyword()
	pingResponse, err := elasticSearch.Ping()
	if err != nil {
		log.Error("Failed to ping Elastic Search: %v", err)
	} else {
		log.Info("Connected to Elastic Search (version: %s)",
			pingResponse.Version.Number)
	}
	appContext.ElasticSearch = elasticSearch
	appContext.EventService = elasticsearch.NewEventService(elasticSearch)
	appContext.AlertQueryService = elasticsearch.NewAlertQueryService(elasticSearch)
	appContext.EventQueryService = elasticsearch.NewEventQueryService(elasticSearch)
	appContext.ReportService = elasticsearch.NewReportService(elasticSearch)

	appContext.Vars.DevWebAppServerUrl = opts.DevServerUri

	var datastoreType string = "elasticsearch"
	if opts.Sqlite {
		datastoreType = "sqlite"
	}

	if datastoreType == "elasticsearch" {
		appContext.DataStore, err = elasticsearch.NewDataStore(elasticSearch)
		if err != nil {
			log.Fatal(err)
		}
	} else if datastoreType == "sqlite" {
		appContext.DataStore, err = sqlite.NewDataStore()
		if err != nil {
			log.Fatal(err)
		}
	}

	httpServer := server.NewServer(appContext)
	err = httpServer.Start(opts.Host + ":" + opts.Port)
	if err != nil {
		log.Fatal(err)
	}
}
示例#5
0
func configure(args []string) Config {
	flagset = flag.NewFlagSet("import", flag.ExitOnError)
	flagset.Usage = usage

	configFilename := flagset.StringP("config", "c", "", "Configuration file")
	verbose := flagset.BoolP("verbose", "v", false, "Verbose output")
	elasticSearchUri := flagset.StringP("elasticsearch", "e", "", "Elastic Search URL")
	username := flagset.StringP("username", "u", "", "Username")
	password := flagset.StringP("password", "p", "", "Password")
	noCheckCertificate := flagset.BoolP("no-check-certificate", "k", false, "Disable certificate check")
	index := flagset.String("index", DEFAULT_INDEX, "Elastic Search index prefix")
	oneshot := flagset.Bool("oneshot", false, "One shot mode (exit on EOF)")
	stdout := flagset.Bool("stdout", false, "Print events to stdout")
	end := flagset.Bool("end", false, "Start at end of file")
	batchSize := flagset.Uint64("batch-size", 1000, "Batch import size")
	useBookmark := flagset.Bool("bookmark", false, "Bookmark location")
	bookmarkPath := flagset.String("bookmark-path", "", "Path to bookmark file")
	noGeoIp := flagset.Bool("no-geoip", false, "Disable GeoIP lookups")
	geoIpDatabase := flagset.String("geoip-database", "", "Path to GeoIP (v2) database file")

	flagset.Parse(args[1:])

	if *verbose {
		log.Info("Setting log level to debug")
		log.SetLevel(log.DEBUG)
	}

	configWrapper := ConfigWrapper{
		Config: Config{
			Index:     DEFAULT_INDEX,
			BatchSize: 1000,
		},
	}

	if *configFilename != "" {
		log.Debug("Loading configuration file %s", *configFilename)
		err := config.LoadConfigTo(*configFilename, &configWrapper)
		if err != nil {
			log.Fatal(err)
		}
	}
	conf := configWrapper.Config

	flagset.Visit(func(flag *flag.Flag) {
		log.Debug("Found command line argument %s -> %s", flag.Name,
			flag.Value.String())
		switch flag.Name {
		case "elasticsearch":
			conf.Url = *elasticSearchUri
		case "username":
			conf.Username = *username
		case "password":
			conf.Password = *password
		case "no-check-certificate":
			conf.DisableCertificateCheck = *noCheckCertificate
		case "index":
			conf.Index = *index
		case "oneshot":
			conf.oneshot = *oneshot
		case "stdout":
			conf.stdout = *stdout
		case "end":
			conf.End = *end
		case "batch-size":
			conf.BatchSize = *batchSize
		case "bookmark":
			conf.Bookmark = *useBookmark
		case "bookmark-path":
			conf.BookmarkPath = *bookmarkPath
		case "no-geoip":
			conf.DisableGeoIp = *noGeoIp
		case "geoip-database":
			conf.GeoIpDatabase = *geoIpDatabase
		case "verbose":
			conf.Verbose = *verbose
		case "config":
		default:
			log.Notice("Unhandle configuration flag %s", flag.Name)
		}
	})

	if len(flagset.Args()) == 1 {
		conf.InputFilename = flagset.Args()[0]
	} else if len(flagset.Args()) > 1 {
		log.Fatal("Multiple input filenames not allowed")
	}

	return conf
}