Exemplo n.º 1
0
func (es *ElasticSearch) SetEventIndex(index string) {
	if strings.HasSuffix(index, "*") {
		baseIndex := strings.TrimSuffix(strings.TrimSuffix(index, "*"), "-")
		es.EventSearchIndex = index
		es.EventBaseIndex = baseIndex
	} else {
		es.EventBaseIndex = index
		es.EventSearchIndex = fmt.Sprintf("%s-*", index)
	}
	log.Info("Event base index: %s", es.EventBaseIndex)
	log.Info("Event search index: %s", es.EventSearchIndex)
}
Exemplo n.º 2
0
func NewService() (*Service, error) {
	args := fmt.Sprintf(
		"dbname=%s user=%s password=%s port=%s sslmode=%s",
		PGDATABASE,
		PGUSER,
		PGPASS,
		PGPORT,
		"disable")
	db, err := sql.Open("postgres", args)
	if err != nil {
		log.Fatal(err)
	}

	var pgVersion string

	err = db.QueryRow("select version()").Scan(&pgVersion)
	if err != nil {
		return nil, err
	}
	log.Info("Connected to PostgreSQL version %s.", pgVersion)

	return &Service{
		db: db,
	}, nil
}
Exemplo n.º 3
0
func (es *ElasticSearch) InitKeyword() error {
	keyword, err := es.GetKeywordType(es.EventBaseIndex)
	if err != nil {
		return err
	}
	es.keyword = keyword
	log.Info("Elastic Search keyword initialized to \"%s\"", es.keyword)
	return nil
}
Exemplo n.º 4
0
func (m *Migrator) Migrate() error {

	var currentVersion int
	nextVersion := 0

	rows, err := m.db.Query("select max(version) from schema")
	if err == nil {
		if rows.Next() {
			if err := rows.Scan(&currentVersion); err != nil {
				return err
			}
			nextVersion = currentVersion + 1
		}
		log.Debug("Current database schema version: %d", currentVersion)
	} else {
		log.Debug("Initializing database.")
	}

	for {

		script, err := resources.AssetString(fmt.Sprintf("sqlite/V%d.sql", nextVersion))
		if err != nil {
			break
		}

		log.Info("Updating database to version %d.", nextVersion)

		tx, err := m.db.Begin()
		if err != nil {
			return err
		}

		_, err = tx.Exec(script)
		if err != nil {
			return err
		}

		err = m.setVersion(tx, nextVersion)
		if err != nil {
			return err
		}

		err = tx.Commit()
		if err != nil {
			return err
		}

		nextVersion++
	}

	return nil
}
Exemplo n.º 5
0
func ReadPipe(pipe io.ReadCloser, doLog bool, logPrefix string) error {
	reader := bufio.NewReader(pipe)
	for {
		line, err := reader.ReadBytes('\n')
		if err != nil && err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		log.Info("%s: %s", logPrefix, strings.TrimSpace(string(line)))
	}
	return nil
}
Exemplo n.º 6
0
func (b *Bookmarker) Init(end bool) error {
	bookmark, err := b.ReadBookmark()
	if err == nil && b.BookmarkIsValid(bookmark) {
		err = b.Reader.SkipTo(bookmark.Offset)
		if err != nil {
			log.Error("Failed to skip to line %d, will skip to end of file: %s", err)
			b.Reader.SkipToEnd()
		}
	} else {
		log.Info("Failed to read bookmark: %s", err)
		if end {
			log.Info("Will start reading at end of file.")
			b.Reader.SkipToEnd()
		} else {
			log.Info("Will start reading at beginning of file.")
		}
	}

	// Test write.
	bookmark = b.GetBookmark()
	return b.WriteBookmark(bookmark)
}
Exemplo n.º 7
0
func Start(directory string) (*exec.Cmd, error) {

	// Get the absolute path if the data directory.
	path, err := filepath.Abs(directory)
	if err != nil {
		return nil, err
	}
	log.Info("Using postgres data directory %s", path)

	command := exec.Command("postgres",
		"-D", path,
		"-c", "log_destination=stderr",
		"-c", "logging_collector=off",
		"-k", path)

	stdout, err := command.StdoutPipe()
	if err != nil {
		log.Error("Failed to open postgres stdout, will not be logged.")
		stdout = nil
	}

	stderr, err := command.StderrPipe()
	if err != nil {
		log.Error("Failed to open postgres stderr, will not be logged.")
		stderr = nil
	}

	err = command.Start()
	if err != nil {
		log.Error("Failed to start postgres: %v", err)
		return nil, err
	}

	if stdout != nil {
		go func() {
			if err := ReadPipe(stdout, true, "postgres stdout"); err != nil {
				log.Error("Failed to read postgres stdout: %v", err)
			}
		}()
	}

	if stderr != nil {
		go func() {
			if err := ReadPipe(stderr, true, "postgres stderr"); err != nil {
				log.Error("Failed to read postgres stderr: %v", err)
			}
		}()
	}

	return command, nil
}
Exemplo n.º 8
0
func (i *BulkEveIndexer) CheckForRedirect() {
	httpClient := http.Client{
		CheckRedirect: func(request *http.Request, via []*http.Request) error {
			if request.Response != nil {
				location, err := request.Response.Location()
				if err == nil {
					log.Info("Redirection to %s detected, updating Elastic Search base URL.", location.String())
					i.baseUrl = location.String()
				}
			}
			return nil
		},
	}
	httpClient.Head(i.es.baseUrl)
}
Exemplo n.º 9
0
func main() {

	// Look for sub-commands, then fall back to server.
	if len(os.Args) > 1 && os.Args[1][0] != '-' {
		switch os.Args[1] {
		case "version":
			VersionMain()
			return
		case "esimport":
			esimport.Main(os.Args[1:])
			return
		case "evereader":
			evereader.Main(os.Args[1:])
			return
		case "server":
			server.Main(os.Args[2:])
			return
		case "pgimport":
			pgimport.Main(os.Args[2:])
			return
		case "sqliteimport":
			sqliteimport.Main(os.Args[2:])
			return
		default:
			log.Fatalf("Unknown command: %s", os.Args[1])
		}
	} else if len(os.Args) > 1 {
		switch os.Args[1] {
		case "-h":
			Usage()
			os.Exit(0)
		}
	}

	log.Info("No command provided, defaulting to server.")
	server.Main(os.Args[1:])
}
Exemplo n.º 10
0
func (s FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var path string

	if r.URL.String() == "/" {
		path = "index.html"
	} else {
		path = strings.TrimPrefix(r.URL.String(), "/")
	}

	// Remove any query string parameters...
	parts := strings.SplitN(path, "?", 2)
	path = parts[0]

	log.Info("Static file request for %s.", path)

	asset := fmt.Sprintf("public/%s", path)
	bytes, err := Asset(asset)
	if err != nil {
		log.Error("Public file not found: %s", path)
		w.WriteHeader(http.StatusNotFound)
	} else {
		w.Write(bytes)
	}
}
Exemplo n.º 11
0
func Main(args []string) {

	var err error

	log.Info("This is EveBox Server version %v (rev: %v)", core.BuildVersion, core.BuildRev)

	flagset := flag.NewFlagSet("server", flag.ExitOnError)

	flagset.StringVarP(&opts.ElasticSearchUri, "elasticsearch", "e", "", "Elastic Search URI (default: http://localhost:9200")
	flagset.StringVarP(&opts.ElasticSearchIndex, "index", "i", "", "Elastic Search Index (default: logstash)")
	flagset.StringVarP(&opts.Port, "port", "p", "5636", "Port to bind to")
	flagset.StringVarP(&opts.Host, "host", "", "0.0.0.0", "Host to bind to")
	flagset.StringVarP(&opts.DevServerUri, "dev", "", "", "Frontend development server URI")
	flagset.BoolVarP(&opts.Version, "version", "", false, "Show version")
	flagset.StringVarP(&opts.Config, "config", "c", "", "Configuration filename")
	flagset.BoolVarP(&opts.NoCheckCertificate, "no-check-certificate", "k", false, "Disable certificate check for Elastic Search")

	flagset.BoolVarP(&opts.Sqlite, "sqlite", "", false, "Use SQLite for the event store")

	flagset.Parse(args[0:])

	if opts.Version {
		VersionMain()
		return
	}

	log.SetLevel(log.DEBUG)

	// If no configuration was provided, see if evebox.yaml exists
	// in the current directory.
	if opts.Config == "" {
		_, err = os.Stat("./evebox.yaml")
		if err == nil {
			opts.Config = "./evebox.yaml"
		}
	}
	if opts.Config != "" {
		log.Printf("Loading configuration file %s.\n", opts.Config)
		conf, err = config.LoadConfig(opts.Config)
		if err != nil {
			log.Fatal(err)
		}
	}

	conf.ElasticSearchIndex = getElasticSearchIndex()
	log.Info("Using ElasticSearch Index %s.", conf.ElasticSearchIndex)

	appContext := server.AppContext{
		Config: conf,
	}
	elasticSearch := elasticsearch.New(getElasticSearchUrl())
	elasticSearch.SetEventIndex(conf.ElasticSearchIndex)
	elasticSearch.InitKeyword()
	pingResponse, err := elasticSearch.Ping()
	if err != nil {
		log.Error("Failed to ping Elastic Search: %v", err)
	} else {
		log.Info("Connected to Elastic Search (version: %s)",
			pingResponse.Version.Number)
	}
	appContext.ElasticSearch = elasticSearch
	appContext.EventService = elasticsearch.NewEventService(elasticSearch)
	appContext.AlertQueryService = elasticsearch.NewAlertQueryService(elasticSearch)
	appContext.EventQueryService = elasticsearch.NewEventQueryService(elasticSearch)
	appContext.ReportService = elasticsearch.NewReportService(elasticSearch)

	appContext.Vars.DevWebAppServerUrl = opts.DevServerUri

	var datastoreType string = "elasticsearch"
	if opts.Sqlite {
		datastoreType = "sqlite"
	}

	if datastoreType == "elasticsearch" {
		appContext.DataStore, err = elasticsearch.NewDataStore(elasticSearch)
		if err != nil {
			log.Fatal(err)
		}
	} else if datastoreType == "sqlite" {
		appContext.DataStore, err = sqlite.NewDataStore()
		if err != nil {
			log.Fatal(err)
		}
	}

	httpServer := server.NewServer(appContext)
	err = httpServer.Start(opts.Host + ":" + opts.Port)
	if err != nil {
		log.Fatal(err)
	}
}
Exemplo n.º 12
0
func Main(args []string) {

	conf := configure(args)

	if conf.BatchSize < 1 {
		log.Fatal("Batch size must be greater than 0")
	}

	if conf.Url == "" {
		log.Error("error: --elasticsearch is a required parameter")
		usage()
		os.Exit(1)
	}

	if conf.InputFilename == "" {
		log.Fatal("error: no input file provided")
	}

	if conf.Bookmark && conf.BookmarkPath == "" {
		conf.BookmarkPath = fmt.Sprintf("%s.bookmark", conf.InputFilename)
		log.Info("Using bookmark file %s", conf.BookmarkPath)
	}

	es := elasticsearch.New(conf.Url)
	es.DisableCertCheck(conf.DisableCertificateCheck)
	if conf.Username != "" || conf.Password != "" {
		if err := es.SetUsernamePassword(conf.Username,
			conf.Password); err != nil {
			log.Fatal("Failed to set username and password: %v", err)
		}
	}
	response, err := es.Ping()
	if err != nil {
		log.Fatal("error: failed to ping Elastic Search:", err)
	}
	log.Info("Connected to Elastic Search v%s (cluster:%s; name: %s)",
		response.Version.Number, response.ClusterName, response.Name)
	majorVersion := response.MajorVersion()

	// Check if the template exists.
	templateExists, err := es.CheckTemplate(conf.Index)
	if !templateExists {
		log.Info("Template %s does not exist, creating...", conf.Index)
		err = es.LoadTemplate(conf.Index, majorVersion)
		if err != nil {
			log.Fatal("Failed to create template:", err)
		}
	} else {
		log.Info("Template %s exists, will not create.", conf.Index)
	}

	var geoipFilter *eve.GeoipFilter
	tagsFilter := &eve.TagsFilter{}

	if !conf.DisableGeoIp {
		geoipdb, err := geoip.NewGeoIpDb(conf.GeoIpDatabase)
		if err != nil {
			log.Notice("Failed to load GeoIP database: %v", err)
		} else {
			log.Info("Using GeoIP database %s, %s", geoipdb.Type(), geoipdb.BuildDate())
			geoipFilter = eve.NewGeoipFilter(geoipdb)
		}
	}

	indexer := elasticsearch.NewIndexer(es, conf.DisableCertificateCheck)
	indexer.IndexPrefix = conf.Index

	reader, err := evereader.New(conf.InputFilename)
	if err != nil {
		log.Fatal(err)
	}

	// Initialize bookmarking...
	var bookmarker *evereader.Bookmarker = nil
	if conf.Bookmark {
		bookmarker = &evereader.Bookmarker{
			Filename: conf.BookmarkPath,
			Reader:   reader,
		}
		err := bookmarker.Init(conf.End)
		if err != nil {
			log.Fatal(err)
		}
	} else if conf.End {
		log.Info("Jumping to end of file.")
		err := reader.SkipToEnd()
		if err != nil {
			log.Fatal(err)
		}
	}

	count := uint64(0)
	lastStatTs := time.Now()
	lastStatCount := uint64(0)
	startTime := time.Now()

	// Number of EOFs in last stat interval.
	eofs := uint64(0)

	go func() {
		err := indexer.Run()
		if err != nil {
			log.Fatal("Elastic Search indexer connection unexpectedly closed:", err)
		} else {
			log.Debug("Indexer exited without issue.")
		}
	}()

	for {
		eof := false
		event, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				eof = true
				eofs++
			} else if _, ok := err.(evereader.MalformedEventError); ok {
				log.Error("Failed to read event but will continue: %v", err)
			} else {
				log.Fatalf("Unrecoverable error reading event: %v", err)
			}
		}

		if event != nil {

			if geoipFilter != nil {
				geoipFilter.AddGeoIP(event)
			}

			tagsFilter.Filter(event)

			if conf.stdout {
				asJson, err := json.Marshal(event)
				if err != nil {
					log.Error("Failed to print event as json: %v", err)
				} else {
					fmt.Println(string(asJson))
				}
			}

			indexer.IndexRawEvent(event)
			count++
		}

		if eof || (count > 0 && count%conf.BatchSize == 0) {
			var bookmark *evereader.Bookmark = nil

			if conf.Bookmark {
				bookmark = bookmarker.GetBookmark()
			}

			response, err := indexer.FlushConnection()
			if err != nil {
				log.Fatal(err)
			}
			if response != nil {
				log.Debug("Indexed %d events {errors=%v}", len(response.Items),
					response.Errors)
			}

			if conf.Bookmark {
				bookmarker.WriteBookmark(bookmark)
			}
		}

		now := time.Now()
		if now.Sub(lastStatTs).Seconds() > 1 && now.Second() == 0 {

			// Calculate the lag in bytes, that is the number of bytes behind
			// the end of file we are.
			lag, err := GetLag(reader)
			if err != nil {
				log.Error("Failed to calculate lag: %v", err)
			}

			log.Info("Total: %d; Last minute: %d; Avg: %.2f/s, EOFs: %d; Lag (bytes): %d",
				count,
				count-lastStatCount,
				float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
				eofs,
				lag)
			lastStatTs = now
			lastStatCount = count
			eofs = 0
		}

		if eof {
			if conf.oneshot {
				break
			} else {
				time.Sleep(1 * time.Second)
			}
		}
	}

	totalTime := time.Since(startTime)

	if conf.oneshot {
		log.Info("Indexed %d events: time=%.2fs; avg=%d/s", count, totalTime.Seconds(),
			uint64(float64(count)/totalTime.Seconds()))
	}
}
Exemplo n.º 13
0
func Main(args []string) {

	var end bool
	var oneshot bool
	var useBookmark bool
	var bookmarkPath string
	var verbose bool

	flagset := flag.NewFlagSet("evereader", flag.ExitOnError)
	flagset.BoolVar(&end, "end", false, "Start at end of file")
	flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
	flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
	flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
	flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
	flagset.Parse(args[1:])

	if verbose {
		log.SetLevel(log.DEBUG)
	}

	if len(flagset.Args()) == 0 {
		log.Fatal("No input files provided.")
	} else if len(flagset.Args()) > 1 {
		log.Fatal("Only one input file allowed.")
	}
	inputFilename := flagset.Args()[0]

	// If useBookmark but no path, set a default.
	if useBookmark && bookmarkPath == "" {
		bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
	}

	encoder := json.NewEncoder(os.Stdout)

	reader, err := evereader.New(flagset.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	// Initialize bookmark.
	var bookmarker *evereader.Bookmarker = nil
	if useBookmark {
		bookmarker = &evereader.Bookmarker{
			Filename: bookmarkPath,
			Reader:   reader,
		}
		err := bookmarker.Init(end)
		if err != nil {
			log.Fatal(err)
		}
	} else if end {
		log.Info("Jumping to end of file.")
		err := reader.SkipToEnd()
		if err != nil {
			log.Fatal(err)
		}
	}

	for {
		eof := false
		event, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				if oneshot {
					break
				}
				eof = true
			} else {
				log.Fatal(err)
			}
		}

		if eof {
			time.Sleep(1 * time.Second)
		} else {
			encoder.Encode(event)

			if useBookmark {
				bookmark := bookmarker.GetBookmark()
				bookmarker.WriteBookmark(bookmark)
			}
		}
	}
}
Exemplo n.º 14
0
func Main(args []string) {

	var end bool
	var oneshot bool
	var useBookmark bool
	var bookmarkPath string
	var verbose bool
	var dbfile string

	flagset := flag.NewFlagSet("sqliteimport", flag.ExitOnError)
	flagset.StringVarP(&dbfile, "database", "D", "", "Database filename")
	flagset.BoolVar(&end, "end", false, "Start at end of file")
	flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
	flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
	flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
	flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
	flagset.Parse(args)

	if verbose {
		log.SetLevel(log.DEBUG)
	}

	if dbfile == "" {
		log.Fatal("Database filename must be provided.")
	}

	if len(flagset.Args()) == 0 {
		log.Fatal("No input files provided.")
	} else if len(flagset.Args()) > 1 {
		log.Fatal("Only one input file allowed.")
	}
	inputFilename := flagset.Args()[0]

	// If useBookmark but no path, set a default.
	if useBookmark && bookmarkPath == "" {
		bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
	}

	reader, err := evereader.New(flagset.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	// Initialize bookmark.
	var bookmarker *evereader.Bookmarker = nil
	if useBookmark {
		bookmarker = &evereader.Bookmarker{
			Filename: bookmarkPath,
			Reader:   reader,
		}
		err := bookmarker.Init(end)
		if err != nil {
			log.Fatal(err)
		}
	} else if end {
		log.Info("Jumping to end of file.")
		err := reader.SkipToEnd()
		if err != nil {
			log.Fatal(err)
		}
	}

	db, err := sqlite.NewSqliteService(dbfile)
	if err != nil {
		log.Fatal(err)
	}
	if err := db.Migrate(); err != nil {
		log.Fatal(err)
	}

	indexer, err := sqlite.NewSqliteIndexer(db)
	if err != nil {
		log.Fatal(err)
	}

	count := uint64(0)
	lastStatTs := time.Now()
	lastStatCount := uint64(0)

	// Number of EOFs in last stat interval.
	eofs := uint64(0)

	tagsFilter := eve.TagsFilter{}

	for {
		eof := false
		event, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				eof = true
				eofs++
			} else {
				log.Fatal(err)
			}
		}

		if event != nil {

			tagsFilter.Filter(event)

			indexer.IndexRawEve(event)
			count++

			if useBookmark {
				bookmark := bookmarker.GetBookmark()
				bookmarker.WriteBookmark(bookmark)
			}
		}

		now := time.Now()

		if now.Sub(lastStatTs).Seconds() > 1 {
			log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
				count,
				count-lastStatCount,
				float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
				eofs)
			lastStatTs = now
			lastStatCount = count
			eofs = 0
			indexer.Flush()
		}

		if eof {
			if oneshot {
				break
			} else {
				indexer.Flush()
				time.Sleep(100 * time.Millisecond)
			}
		}

	}

	now := time.Now()

	log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
		count,
		count-lastStatCount,
		float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
		eofs)

	indexer.Flush()
}
Exemplo n.º 15
0
func configure(args []string) Config {
	flagset = flag.NewFlagSet("import", flag.ExitOnError)
	flagset.Usage = usage

	configFilename := flagset.StringP("config", "c", "", "Configuration file")
	verbose := flagset.BoolP("verbose", "v", false, "Verbose output")
	elasticSearchUri := flagset.StringP("elasticsearch", "e", "", "Elastic Search URL")
	username := flagset.StringP("username", "u", "", "Username")
	password := flagset.StringP("password", "p", "", "Password")
	noCheckCertificate := flagset.BoolP("no-check-certificate", "k", false, "Disable certificate check")
	index := flagset.String("index", DEFAULT_INDEX, "Elastic Search index prefix")
	oneshot := flagset.Bool("oneshot", false, "One shot mode (exit on EOF)")
	stdout := flagset.Bool("stdout", false, "Print events to stdout")
	end := flagset.Bool("end", false, "Start at end of file")
	batchSize := flagset.Uint64("batch-size", 1000, "Batch import size")
	useBookmark := flagset.Bool("bookmark", false, "Bookmark location")
	bookmarkPath := flagset.String("bookmark-path", "", "Path to bookmark file")
	noGeoIp := flagset.Bool("no-geoip", false, "Disable GeoIP lookups")
	geoIpDatabase := flagset.String("geoip-database", "", "Path to GeoIP (v2) database file")

	flagset.Parse(args[1:])

	if *verbose {
		log.Info("Setting log level to debug")
		log.SetLevel(log.DEBUG)
	}

	configWrapper := ConfigWrapper{
		Config: Config{
			Index:     DEFAULT_INDEX,
			BatchSize: 1000,
		},
	}

	if *configFilename != "" {
		log.Debug("Loading configuration file %s", *configFilename)
		err := config.LoadConfigTo(*configFilename, &configWrapper)
		if err != nil {
			log.Fatal(err)
		}
	}
	conf := configWrapper.Config

	flagset.Visit(func(flag *flag.Flag) {
		log.Debug("Found command line argument %s -> %s", flag.Name,
			flag.Value.String())
		switch flag.Name {
		case "elasticsearch":
			conf.Url = *elasticSearchUri
		case "username":
			conf.Username = *username
		case "password":
			conf.Password = *password
		case "no-check-certificate":
			conf.DisableCertificateCheck = *noCheckCertificate
		case "index":
			conf.Index = *index
		case "oneshot":
			conf.oneshot = *oneshot
		case "stdout":
			conf.stdout = *stdout
		case "end":
			conf.End = *end
		case "batch-size":
			conf.BatchSize = *batchSize
		case "bookmark":
			conf.Bookmark = *useBookmark
		case "bookmark-path":
			conf.BookmarkPath = *bookmarkPath
		case "no-geoip":
			conf.DisableGeoIp = *noGeoIp
		case "geoip-database":
			conf.GeoIpDatabase = *geoIpDatabase
		case "verbose":
			conf.Verbose = *verbose
		case "config":
		default:
			log.Notice("Unhandle configuration flag %s", flag.Name)
		}
	})

	if len(flagset.Args()) == 1 {
		conf.InputFilename = flagset.Args()[0]
	} else if len(flagset.Args()) > 1 {
		log.Fatal("Multiple input filenames not allowed")
	}

	return conf
}
Exemplo n.º 16
0
func (h ApiWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	response := h.handler.ServeHTTP(h.appContext, r)
	if response != nil {
		switch response := response.(type) {
		case error:

			log.Error("%+v", response)

			var message string

			switch cause := errors.Cause(response).(type) {
			case *elasticsearch.DatastoreError:
				message = cause.Message
			}

			if message == "" {
				message = response.Error()
			}

			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(http.StatusBadRequest)
			encoder := json.NewEncoder(w)
			encoder.Encode(HttpStatusResponseBody{
				StatusCode: http.StatusBadRequest,
				Message:    message,
			})
		case HttpResponse:
			statusCode := http.StatusOK
			contentType := "application/json"

			// Set status code if provided.
			if response.statusCode != 0 {
				statusCode = response.statusCode
			}

			// Set content type if provided.
			if response.contentType != "" {
				contentType = response.contentType
			}

			// Merge in provided headers.
			if response.headers != nil {
				for key, val := range response.headers {
					log.Info("Setting %s -> %s", key, val)
					w.Header().Set(key, val)
				}
			}

			w.Header().Set("Content-Type", contentType)
			w.WriteHeader(statusCode)
			if response.body != nil {
				switch body := response.body.(type) {
				case []byte:
					w.Write(body)
				default:
					encoder := json.NewEncoder(w)
					encoder.Encode(response.body)
				}
			}
		default:
			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(http.StatusOK)
			encoder := json.NewEncoder(w)
			encoder.Encode(response)
		}
	}
}
Exemplo n.º 17
0
// bulkUpdateTags will add and/or remvoe tags from a set of documents using
// the Elastic Search bulk API.
func bulkUpdateTags(es *ElasticSearch, documents []map[string]interface{},
	addTags []string, rmTags []string) (bool, error) {

	bulk := make([]string, 0)

	for _, item := range documents {
		doc := JsonMap(item)

		currentTags := doc.GetMap("_source").GetAsStrings("tags")
		tags := make([]string, 0)

		for _, tag := range currentTags {
			if rmTags == nil || !StringSliceContains(rmTags, tag) {
				tags = append(tags, tag)
			}
		}

		for _, tag := range addTags {
			if !StringSliceContains(tags, tag) {
				tags = append(tags, tag)
			}
		}

		id := doc.Get("_id").(string)
		docType := doc.Get("_type").(string)
		index := doc.Get("_index").(string)

		command := m{
			"update": m{
				"_id":    id,
				"_type":  docType,
				"_index": index,
			},
		}
		bulk = append(bulk, util.ToJson(command))

		partial := m{
			"doc": m{
				"tags": tags,
			},
		}
		bulk = append(bulk, util.ToJson(partial))
	}

	// Needs to finish with a new line.
	bulk = append(bulk, "")
	bulkString := strings.Join(bulk, "\n")
	response, err := es.HttpClient.PostString("_bulk", "application/json", bulkString)
	if err != nil {
		log.Error("Failed to update event tags: %v", err)
		return false, err
	}

	retry := false

	if response.StatusCode != http.StatusOK {
		return retry, NewElasticSearchError(response)
	} else {
		bulkResponse := BulkResponse{}

		if err := es.Decode(response, &bulkResponse); err != nil {
			log.Error("Failed to decode bulk response: %v", err)
		} else {
			log.Info("Tags updated on %d events; errors=%v",
				len(bulkResponse.Items), bulkResponse.Errors)
			if bulkResponse.Errors {
				retry = true
				for _, item := range bulkResponse.Items {
					logBulkUpdateError(item)
				}
			}
		}
	}

	return retry, nil
}