Exemple #1
0
func ReadApiConfig(fileLocation string) (a *ApiConfig) {
	f := flag.NewFlagSet("api", flag.ContinueOnError)

	flagApiHostname := f.String("api_host", "127.0.0.1", "Api hostname")
	flagApiPort := f.Uint("api_port", 8844, "Api port")

	globalconf.Register("api", f)

	g, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename: fileLocation,
	})

	if err != nil {
		log.Printf("[config][warning] Could not parse config file %s"+
			" Error: %s, Falling back to default settings",
			fileLocation, err.Error())
	} else {
		g.ParseAll()
	}

	return &ApiConfig{
		Host: *flagApiHostname,
		Port: uint16(*flagApiPort),
	}
}
Exemple #2
0
func main() {
	// Get the configuration
	//
	if *configFile == "" {
		*configFile = "./nightcrawler.ini"
	}
	opts := globalconf.Options{Filename: *configFile, EnvPrefix: "NC_"}
	conf, _ := globalconf.NewWithOptions(&opts)
	conf.ParseAll()

	// Static assets should be served by nginx in production
	//
	println(martini.Env)
	m := martini.Classic()
	if martini.Env != "production" {
		m.Use(martini.Static("assets"))
	}

	//  This is just a temp route in development mode
	//
	if martini.Env == "development" {
		m.Get("/", UserRendererMiddleware(), func(r render.Render) {
			r.HTML(200, "root", nil)
		})
	}

	// Top level of app
	// This section is everything outside of authentication
	//
	m.Group("/app", func(r martini.Router) {
		r.Get("/sign_in", controllers.SignIn)
		r.Post("/sign_in", csrf.Validate, controllers.SignIn)
		r.Get("/sign_up", controllers.SignUp)
		r.Post("/sign_up", csrf.Validate, controllers.CreateSignUp)
		r.Get("/sign_out", controllers.SignOut)
		r.NotFound(func(x render.Render) {
			x.HTML(404, "404", nil)
		})
	},
		CookieStore(*userCookieSecret, *userSessionName),
		UserSessionAuth(),
		CSRFMiddleware(*userSessionSecret, *userSessionKey),
		UserRendererMiddleware(),
	)

	// Admin interface
	m.Group("/app/admin", func(r martini.Router) {
		r.Get("/sign_in", controllers.AdminSignIn)
		r.NotFound(func(x render.Render) {
			x.HTML(404, "404_admin", nil)
		})
	},
		CookieStore("admin123", "nightcrawler_admin"),
		AdminSessionAuth(),
		CSRFMiddleware("admin123", "adminId"),
		AdminRendererMiddleware(),
	)

	m.Run()
}
Exemple #3
0
func setup(name string) error {
	var (
		conf *globalconf.GlobalConf
		file string
		err  error
	)

	// Check for environment variable override, and return error if override is set but file is
	// inaccessible. Otherwise, try for the default global location (in the "/etc" directory).
	if file := os.Getenv(strings.ToUpper(name) + "_CONFIG"); file != "" {
		if _, err = os.Stat(file); err != nil {
			return err
		}
	} else {
		file = "/etc/" + name + "/" + name + ".conf"
		if _, err = os.Stat(file); err != nil {
			file = ""
		}
	}

	// Load from specific configuration file if set, or use local configuration file as a fallback.
	if file != "" {
		options := &globalconf.Options{Filename: file, EnvPrefix: ""}
		if conf, err = globalconf.NewWithOptions(options); err != nil {
			return err
		}
	} else if conf, err = globalconf.New(name); err != nil {
		return err
	}

	conf.EnvPrefix = strings.ToUpper(name) + "_"
	conf.ParseAll()

	return nil
}
Exemple #4
0
func ReadDbConfig(fileLocation string) *DatabaseConfig {
	f := flag.NewFlagSet("database", flag.ContinueOnError)

	flagDBHostname := f.String("db_host", "127.0.0.1", "Database hostname")
	flagDBPort := f.Uint("db_port", 5432, "Database port")
	flagDBUsername := f.String("db_username", "ubi", "Database username")
	flagDBPassword := f.String("db_password", "password", "Database password")
	flagDBName := f.String("db_name", "ubi_course", "Database name")
	flagDBSSLMode := f.String("db_sslmode", "disable", "Database SSL mode")

	globalconf.Register("database", f)

	g, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename: fileLocation,
	})

	if err != nil {
		log.Printf("[config][warning] Could not parse config file '%s': %s. "+
			"Falling back to default settings",
			fileLocation, err.Error())
	} else {
		g.ParseAll()
	}
	return &DatabaseConfig{
		Hostname: *flagDBHostname,
		Port:     uint16(*flagDBPort),
		Username: *flagDBUsername,
		Password: *flagDBPassword,
		Name:     *flagDBName,
		SSLMode:  *flagDBSSLMode,
	}
}
Exemple #5
0
func ReadDaqConfig(fileLocation string) *DaqConfig {
	f := flag.NewFlagSet("daq", flag.ContinueOnError)

	flagDaqInterval := f.Uint("daq_interval", 5, "Daq interval in minutes")
	flagMockerOn := f.Bool("daq_mocker_on", false, "Mocker state")

	globalconf.Register("daq", f)

	g, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename: fileLocation,
	})

	if err != nil {
		log.Printf("[config][warning] Could not parse config file %s"+
			" Error: %s, Falling back to default settings",
			fileLocation, err.Error())
	} else {
		g.ParseAll()
	}

	return &DaqConfig{
		Interval: *flagDaqInterval,
		MockerOn: *flagMockerOn,
	}
}
Exemple #6
0
// parse conf from env and args
func parseConf() {

	// let mflag parse first
	mflag.Parse()

	// using gconf parse env
	gconf, err := globalconf.NewWithOptions(&globalconf.Options{
		EnvPrefix: "WICKET_",
	})

	if err != nil {
		log.Fatalf("error parsing config file: %v", err)
	}

	fs := flag.NewFlagSet("", flag.ContinueOnError)

	mflag.VisitAll(func(f *mflag.Flag) {
		for _, n := range f.Names {
			if len(n) < 2 {
				continue
			}

			n = strings.TrimPrefix(n, "-")
			fs.Var(f.Value, n, f.Usage)
		}
	})

	gconf.ParseSet("", fs)
}
Exemple #7
0
func ReadConfig(filename string) Config {

	dbFlagSet := flag.NewFlagSet("mysql", flag.ExitOnError)

	flagUser := dbFlagSet.String("username", "", "Database username")
	flagPass := dbFlagSet.String("password", "", "Database password")
	flagHost := dbFlagSet.String("host", "localhost", "Database host")
	flagDbName := dbFlagSet.String("database", "", "Database name")

	globalconf.Register("mysql", dbFlagSet)

	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename: filename,
	})

	if err != nil {
		log.Fatalf(err.Error())
	}

	conf.ParseAll()

	return Config{
		Username: *flagUser,
		Host:     *flagHost,
		Password: *flagPass,
		Database: *flagDbName,
	}
}
Exemple #8
0
func GetConfig() (*Config, error) {
	c := &Config{}
	var keyFile string
	var keyContents string
	var proxyProtoHttpsPorts string
	flag.StringVar(&keyFile, "jwt-public-key-file", "", "Location of the public-key used to validate JWTs.")
	flag.StringVar(&keyContents, "jwt-public-key-contents", "", "An alternative to jwt-public-key-file. The contents of the key.")
	flag.StringVar(&c.ListenAddr, "listen-address", ":8080", "The tcp address to listen on.")
	flag.StringVar(&c.CattleAddr, "cattle-address", "", "The tcp address to forward cattle API requests to. Will not proxy to cattle api if this option is not provied.")
	flag.IntVar(&c.ParentPid, "parent-pid", 0, "If provided, this process will exit when the specified parent process stops running.")
	flag.StringVar(&proxyProtoHttpsPorts, "https-proxy-protocol-ports", "", "If proxy protocol is used, a list of proxy ports that will allow us to recognize that the connection was over https.")

	confOptions := &globalconf.Options{
		EnvPrefix: "PROXY_",
	}

	conf, err := globalconf.NewWithOptions(confOptions)

	if err != nil {
		return nil, err
	}

	conf.ParseAll()

	if keyFile != "" && keyContents != "" {
		return nil, fmt.Errorf("Can't specify both jwt-public-key-file and jwt-public-key-contents")
	}
	var parsedKey interface{}
	var parseErr error
	if keyFile != "" {
		parsedKey, parseErr = ParsePublicKey(keyFile)
	} else if keyContents != "" {
		parsedKey, parseErr = ParsePublicKeyFromMemory(keyContents)
	} else {
		parseErr = fmt.Errorf("Must specify one of jwt-public-key-file and jwt-public-key-contents")
	}
	if parseErr != nil {
		return nil, parseErr
	}

	c.PublicKey = parsedKey

	portMap := make(map[int]bool)
	ports := strings.Split(proxyProtoHttpsPorts, ",")
	for _, port := range ports {
		if p, err := strconv.Atoi(port); err == nil {
			portMap[p] = true
		}
	}
	c.ProxyProtoHttpsPorts = portMap

	return c, nil
}
Exemple #9
0
func Parse() error {
	flag.BoolVar(&Config.HaProxyMonitor, "haproxy-monitor", false, "Monitor HAProxy")
	flag.IntVar(&Config.Port, "port", 8080, "Listen port")
	flag.StringVar(&Config.Ip, "ip", "", "Listen IP, defaults to all IPs")
	flag.StringVar(&Config.CAdvisorUrl, "cadvisor-url", "http://localhost:8081", "cAdvisor URL")
	flag.StringVar(&Config.DockerUrl, "docker-host", "unix:///var/run/docker.sock", "Docker host URL")
	flag.IntVar(&Config.NumStats, "num-stats", 600, "Number of stats to show by default")
	flag.BoolVar(&Config.Auth, "auth", false, "Authenticate requests")
	flag.StringVar(&Config.HostUuid, "host-uuid", "", "Host UUID")
	flag.BoolVar(&Config.HostUuidCheck, "host-uuid-check", true, "Validate host UUID")
	flag.StringVar(&Config.Key, "public-key", "", "Public Key for Authentication")
	flag.IntVar(&Config.EventsPoolSize, "events-pool-size", 10, "Size of worker pool for processing docker events.")
	flag.StringVar(&Config.CattleUrl, "cattle-url", "", "URL for accessing cattle api")
	flag.StringVar(&Config.CattleAccessKey, "cattle-access-key", "", "Access key for cattle api")
	flag.StringVar(&Config.CattleSecretKey, "cattle-secret-key", "", "Secret key for cattle api")
	flag.StringVar(&Config.CattleStateDir, "cattle-state-dir", "", "Directory where Rancher state is persisted.")
	flag.StringVar(&Config.PidFile, "pid-file", "", "PID file")
	flag.StringVar(&Config.LogFile, "log", "", "Log file")

	confOptions := &globalconf.Options{
		EnvPrefix: "HOST_API_",
	}

	filename := os.Getenv("HOST_API_CONFIG_FILE")
	if len(filename) > 0 {
		confOptions.Filename = filename
	}

	conf, err := globalconf.NewWithOptions(confOptions)

	if err != nil {
		return err
	}

	conf.ParseAll()

	if len(Config.Key) > 0 {
		if err := ParsedPublicKey(); err != nil {
			glog.Error("Error reading file")
			return err
		}
	}

	s, err := os.Stat("/run/systemd/system")
	if err != nil || !s.IsDir() {
		Config.Systemd = false
	} else {
		Config.Systemd = true
	}

	return nil
}
Exemple #10
0
// TODO:  html template
// TODO:  email this out
// TODO:  better comments
func main() {
	// loading configuration
	flag.Parse()
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename: config,
	})
	if err != nil {
		log.Fatal(err)
	}
	conf.ParseAll()

	ccAPI, _ := api.NewAPI(config, authToken, apiKey, daysBack, debug)

	// Generate a report from a stored file
	if prevReport != "" {
		c := types.Load(prevReport)
		c.BuildCampaignReport(getTopDomains, getTopClicks)
		fmt.Println(types.Render(c))
		os.Exit(0)
	}

	runTime := time.Now().Format("2006-01-02T15:04")
	saveTo := fmt.Sprintf("./logs/%s.gob", runTime)

	log.Print(runTime, saveTo)

	camps, err := ccAPI.GetCampaigns()
	if err != nil {
		log.Fatal(err)
	}
	log.Print(camps)
	log.Print("retrieved campaigns: ", len(camps.Campaigns))

	for _, c := range camps.Campaigns {
		err = ccAPI.GetCampaignDetail(c)
		log.Print(err)
		log.Print(c)

		err = ccAPI.GetCampaignPreview(c)
		log.Print(err)

		err = ccAPI.GetCampaignTracking(c)

		b, _ := json.MarshalIndent(c, "", "  ")
		fmt.Println(string(b))
	}
	types.Save(camps, saveTo)
	camps.BuildCampaignReport(getTopDomains, getTopClicks)
	fmt.Println(camps)
	fmt.Println(types.Render(camps))
}
Exemple #11
0
func init() {
	chanmap = make(map[string]*hbot.IrcChannel)
	config, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename: configFile,
	})
	if err != nil {
		log.Fatalf("Cannot parse configuration file: %s", err)
	}
	globalconf.Register("tumblr", tumblrConf)
	globalconf.Register("twitter", twitterConf)
	globalconf.Register("logger", loggerConf)
	globalconf.Register("commands", commandsConf)
	config.ParseAll()
}
Exemple #12
0
func ConfInit() {
	flag.Parse()
	if *g_conf_file == "" {
		//panic("please set conf file ")
		println("have no config file")
		return
	}
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename: *g_conf_file,
	})

	if err != nil {
		panic(err)
	}
	conf.ParseAll()
}
Exemple #13
0
func getConfig(flagset *flag.FlagSet, userCfgFile string) (*config.Config, error) {
	opts := globalconf.Options{EnvPrefix: "API_HOSTD_"}

	if userCfgFile != "" {
		// Fail hard if a user-provided config is not usable
		fi, err := os.Stat(userCfgFile)
		if err != nil {
			ctxLog.Fatalf("Unable to use config file %s: %v", userCfgFile, err)
		}
		if fi.IsDir() {
			ctxLog.Fatalf("Provided config %s is a directory, not a file", userCfgFile)
		}
		opts.Filename = userCfgFile
	} else if _, err := os.Stat(DefaultConfigFile); err == nil {
		opts.Filename = DefaultConfigFile
	}

	gconf, err := globalconf.NewWithOptions(&opts)
	if err != nil {
		return nil, err
	}

	gconf.ParseSet("", flagset)

	cfg := config.Config{
		Verbosity: (*flagset.Lookup("verbosity")).Value.(flag.Getter).Get().(int),
		IP:        (*flagset.Lookup("ip")).Value.(flag.Getter).Get().(string),
		Port:      (*flagset.Lookup("port")).Value.(flag.Getter).Get().(string),
		Secret:    (*flagset.Lookup("jwt_sign_key")).Value.(flag.Getter).Get().(string),

		CORSAllowedOrigins:     config.StringToSlice((*flagset.Lookup("cors_allowed_origins")).Value.(flag.Getter).Get().(string)),
		CORSAllowedMethods:     config.StringToSlice((*flagset.Lookup("cors_allowed_methods")).Value.(flag.Getter).Get().(string)),
		CORSAllowedHeaders:     config.StringToSlice((*flagset.Lookup("cors_allowed_headers")).Value.(flag.Getter).Get().(string)),
		CORSExposedHeaders:     config.StringToSlice((*flagset.Lookup("cors_exposed_headers")).Value.(flag.Getter).Get().(string)),
		CORSAllowCredentials:   (*flagset.Lookup("cors_allow_credentials")).Value.(flag.Getter).Get().(bool),
		CORSMaxAge:             (*flagset.Lookup("cors_max_age")).Value.(flag.Getter).Get().(int),
		CORSOptionsPassThrough: (*flagset.Lookup("cors_options_pass_through")).Value.(flag.Getter).Get().(bool),
		CORSDebug:              (*flagset.Lookup("cors_debug")).Value.(flag.Getter).Get().(bool),
	}

	log.SetLevel(log.Level(cfg.Verbosity))

	ctxLog.Infof("Loaded config: [%+v]", cfg)

	return &cfg, nil
}
Exemple #14
0
// Read config
//
// Initialize Config from Config File
func ReadConfig(ConfigFile string, Datadir string, EnvPrefix string) *ConfigManager {
	if !FileExist(ConfigFile) {
		// create ConfigFile if it does not exist, otherwise
		// globalconf will panic when trying to persist flags.
		fmt.Printf("config file '%s' doesn't exist, creating it\n", ConfigFile)
		os.Create(ConfigFile)
	}
	g, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename:  ConfigFile,
		EnvPrefix: EnvPrefix,
	})
	if err != nil {
		fmt.Println(err)
	} else {
		g.ParseAll()
	}
	cfg := &ConfigManager{ExecPath: Datadir, Debug: true, conf: g, Paranoia: true}
	return cfg
}
Exemple #15
0
func GetConfig() (*Config, error) {
	c := &Config{}
	var keyFile string
	var keyContents string
	flag.StringVar(&keyFile, "jwt-public-key-file", "", "Location of the public-key used to validate JWTs.")
	flag.StringVar(&keyContents, "jwt-public-key-contents", "", "An alternative to jwt-public-key-file. The contents of the key.")
	flag.StringVar(&c.ListenAddr, "listen-address", "localhost:8080", "The tcp address to listen on.")
	flag.StringVar(&c.CattleAddr, "cattle-address", "", "The tcp address to forward cattle API requests to. Will not proxy to cattle api if this option is not provied.")
	flag.IntVar(&c.ParentPid, "parent-pid", 0, "If provided, this process will exit when the specified parent process stops running.")

	confOptions := &globalconf.Options{
		EnvPrefix: "PROXY_",
	}

	conf, err := globalconf.NewWithOptions(confOptions)

	if err != nil {
		return nil, err
	}

	conf.ParseAll()

	if keyFile != "" && keyContents != "" {
		return nil, fmt.Errorf("Can't specify both jwt-public-key-file and jwt-public-key-contents")
	}
	var parsedKey interface{}
	var parseErr error
	if keyFile != "" {
		parsedKey, parseErr = ParsePublicKey(keyFile)
	} else if keyContents != "" {
		parsedKey, parseErr = ParsePublicKeyFromMemory(keyContents)
	} else {
		parseErr = fmt.Errorf("Must specify one of jwt-public-key-file and jwt-public-key-contents")
	}
	if parseErr != nil {
		return nil, parseErr
	}

	c.PublicKey = parsedKey
	return c, nil
}
Exemple #16
0
func ReadMockerConfig(fileLocation string) *MockerConfig {
	f := flag.NewFlagSet("mocker", flag.ContinueOnError)

	flagMockerAmount := f.Uint("mocker_place_amount", 5, "Amount of mocked places")
	flagMockerRecordsMin := f.Uint("mocker_records_min", 0, "Minimum amount of queuers")
	flagMockerRecordsMax := f.Uint("mocker_records_max", 5, "Maximum amount of queuers")
	flagMockerTimeMin := f.Uint("mocker_time_min", 1, "Minimum time in queue")
	flagMockerTimeMax := f.Uint("mocker_time_max", 50, "Maximum time in queue")
	flagMockerHistoric := f.Bool("mocker_historic", false, "Enable historic data generation")
	flagMockerHistoricWeeks := f.Uint("mocker_historic_weeks", 0, "Number of weeks to generate historic data")
	flagMockerHistoricInterval := f.Uint("mocker_historic_interval", 15, "Interval for gistoric records")

	globalconf.Register("mocker", f)

	g, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename: fileLocation,
	})

	if err != nil {
		log.Printf("[config][warning] Could not parse config file %s"+
			" Error: %s, Falling back to default settings",
			fileLocation, err.Error())
	} else {
		g.ParseAll()
	}

	return &MockerConfig{
		PlaceAmount:      *flagMockerAmount,
		RecordsMin:       *flagMockerRecordsMin,
		RecordsMax:       *flagMockerRecordsMax,
		TimeMin:          *flagMockerTimeMin,
		TimeMax:          *flagMockerTimeMax,
		Historic:         *flagMockerHistoric,
		HistoricWeek:     *flagMockerHistoricWeeks,
		HistoricInterval: *flagMockerHistoricInterval,
	}
}
func main() {
	flag.Parse()

	// Only try and parse the conf file if it exists
	if _, err := os.Stat(*confFile); err == nil {
		conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
		if err != nil {
			log.Fatal(4, err.Error())
		}
		conf.ParseAll()
	}

	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))

	if *showVersion {
		fmt.Println("nsq_probe_events_to_elasticsearch")
		return
	}

	if *channel == "" {
		rand.Seed(time.Now().UnixNano())
		*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
	}

	if *topic == "" {
		log.Fatal(4, "--topic is required")
	}

	if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
		log.Fatal(4, "--nsqd-tcp-address or --lookupd-http-address required")
	}
	if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
		log.Fatal(4, "use --nsqd-tcp-address or --lookupd-http-address not both")
	}

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(4, err.Error())
	}
	metrics, err := helper.New(true, *statsdAddr, *statsdType, "nsq_probe_events_to_elasticsearch", strings.Replace(hostname, ".", "_", -1))
	if err != nil {
		log.Fatal(4, err.Error())
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	eventsToEsOK = metrics.NewCount("events_to_es.ok")
	eventsToEsFail = metrics.NewCount("events_to_es.fail")
	messagesSize = metrics.NewMeter("message_size", 0)
	msgsAge = metrics.NewMeter("message_age", 0)
	esPutDuration = metrics.NewTimer("es_put_duration", 0)
	msgsHandleOK = metrics.NewCount("handle.ok")
	msgsHandleFail = metrics.NewCount("handle.fail")

	cfg := nsq.NewConfig()
	cfg.UserAgent = "nsq_probe_events_to_elasticsearch"
	err = app.ParseOpts(cfg, *consumerOpts)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	cfg.MaxInFlight = *maxInFlight

	consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", metrics)

	if err != nil {
		log.Fatal(4, err.Error())
	}

	handler, err := NewESHandler()
	if err != nil {
		log.Fatal(4, err.Error())
	}

	consumer.AddConcurrentHandlers(handler, 80)

	nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
	if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
		nsqdAdds = []string{}
	}
	err = consumer.ConnectToNSQDs(nsqdAdds)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	log.Info("connected to nsqd")

	lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
	if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
		lookupdAdds = []string{}
	}
	err = consumer.ConnectToNSQLookupds(lookupdAdds)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	go func() {
		log.Info("INFO starting listener for http/debug on %s", *listenAddr)
		httperr := http.ListenAndServe(*listenAddr, nil)
		if httperr != nil {
			log.Info(httperr.Error())
		}
	}()

	for {
		select {
		case <-consumer.StopChan:
			return
		case <-sigChan:
			consumer.Stop()
		}
	}
}
Exemple #18
0
func main() {
	flag.Parse()
	// Set 'cfile' here if *confFile exists, because we should only try and
	// parse the conf file if it exists. If we try and parse the default
	// conf file location when it's not there, we (unsurprisingly) get a
	// panic.
	var cfile string
	if _, err := os.Stat(*confFile); err == nil {
		cfile = *confFile
	}

	// Still parse globalconf, though, even if the config file doesn't exist
	// because we want to be able to use environment variables.
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename:  cfile,
		EnvPrefix: "TASKAGENT_",
	})
	if err != nil {
		panic(fmt.Sprintf("error with configuration file: %s", err))
	}
	conf.ParseAll()

	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
	// workaround for https://github.com/grafana/grafana/issues/4055
	switch *logLevel {
	case 0:
		log.Level(log.TRACE)
	case 1:
		log.Level(log.DEBUG)
	case 2:
		log.Level(log.INFO)
	case 3:
		log.Level(log.WARN)
	case 4:
		log.Level(log.ERROR)
	case 5:
		log.Level(log.CRITICAL)
	case 6:
		log.Level(log.FATAL)
	}

	if *showVersion {
		fmt.Printf("task-agent (built with %s, git hash %s)\n", runtime.Version(), GitHash)
		return
	}

	if *nodeName == "" {
		log.Fatal(4, "name must be set.")
	}

	snapUrl, err := url.Parse(*snapUrlStr)
	if err != nil {
		log.Fatal(4, "could not parse snapUrl. %s", err)
	}
	snapClient, err := snap.NewClient(*nodeName, *tsdbAddr, *apiKey, snapUrl)
	if err != nil {
		log.Fatal(4, err.Error())
	}

	InitTaskCache(snapClient)

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)
	shutdownStart := make(chan struct{})

	controllerUrl, err := url.Parse(*serverAddr)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	controllerUrl.Path = path.Clean(controllerUrl.Path + fmt.Sprintf("/socket/%s/%d", *nodeName, Version))

	if controllerUrl.Scheme != "ws" && controllerUrl.Scheme != "wss" {
		log.Fatal(4, "invalid server address.  scheme must be ws or wss. was %s", controllerUrl.Scheme)
	}

	conn, err := connect(controllerUrl)
	if err != nil {
		log.Fatal(4, "unable to connect to server on url %s: %s", controllerUrl.String(), err)
	}

	//create new session, allow 1000 events to be queued in the writeQueue before Emit() blocks.
	sess := session.NewSession(conn, 1000)
	sess.On("disconnect", func() {
		// on disconnect, reconnect.
		ticker := time.NewTicker(time.Second)
		connected := false
		for !connected {
			select {
			case <-shutdownStart:
				ticker.Stop()
				return
			case <-ticker.C:
				conn, err := connect(controllerUrl)
				if err == nil {
					sess.Conn = conn
					connected = true
					go sess.Start()
				}
			}
		}
		ticker.Stop()
	})

	sess.On("heartbeat", func(body []byte) {
		log.Debug("recieved heartbeat event. %s", body)
	})

	sess.On("taskList", HandleTaskList())
	sess.On("taskUpdate", HandleTaskUpdate())
	sess.On("taskAdd", HandleTaskAdd())
	sess.On("taskRemove", HandleTaskRemove())

	go sess.Start()

	//periodically send an Updated Catalog.
	go SendCatalog(sess, snapClient, shutdownStart)

	// connect to the snap server and monitor that it is up.
	go snapClient.Run()

	//wait for interupt Signal.
	<-interrupt
	log.Info("interrupt")
	close(shutdownStart)
	sess.Close()
	return
}
Exemple #19
0
func initConfig() {
	configfile := mflag.String([]string{"-config"}, "/etc/sshpiperd.conf", "Config file path. Note: any option will be overwrite if it is set by commandline")

	mflag.StringVar(&config.ListenAddr, []string{"l", "-listen_addr"}, "0.0.0.0", "Listening Address")
	mflag.UintVar(&config.Port, []string{"p", "-port"}, 2222, "Listening Port")
	mflag.StringVar(&config.WorkingDir, []string{"w", "-working_dir"}, "/var/sshpiper", "Working Dir")
	mflag.StringVar(&config.PiperKeyFile, []string{"i", "-server_key"}, "/etc/ssh/ssh_host_rsa_key", "Key file for SSH Piper")
	mflag.StringVar(&config.Challenger, []string{"c", "-challenger"}, "", "Additional challenger name, e.g. pam, emtpy for no additional challenge")
	mflag.StringVar(&config.Logfile, []string{"-log"}, "", "Logfile path. Leave emtpy or any error occurs will fall back to stdout")
	mflag.BoolVar(&config.AllowBadUsername, []string{"-allow_bad_username"}, false, "disable username check while search the working dir")
	mflag.BoolVar(&config.ShowHelp, []string{"h", "-help"}, false, "Print help and exit")
	mflag.BoolVar(&config.ShowVersion, []string{"-version"}, false, "Print version and exit")

	mflag.Parse()

	if _, err := os.Stat(*configfile); os.IsNotExist(err) {
		if !mflag.IsSet("-config") {
			*configfile = ""
		} else {
			logger.Fatalf("config file %v not found", *configfile)
		}
	}

	gconf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename:  *configfile,
		EnvPrefix: "SSHPIPERD_",
	})

	if err != nil { // this error will happen only if file error
		logger.Fatalln("load config file error %v: %v", *configfile, err)
	}

	// build a dummy flag set for globalconf to parse
	fs := flag.NewFlagSet("", flag.ContinueOnError)

	ignoreSet := make(map[string]bool)
	mflag.Visit(func(f *mflag.Flag) {
		for _, n := range f.Names {
			ignoreSet[n] = true
		}
	})

	// should be ignored
	ignoreSet["-help"] = true
	ignoreSet["-version"] = true

	mflag.VisitAll(func(f *mflag.Flag) {
		for _, n := range f.Names {
			if len(n) < 2 {
				continue
			}

			if !ignoreSet[n] {
				n = strings.TrimPrefix(n, "-")
				fs.Var(f.Value, n, f.Usage)
			}
		}
	})

	gconf.ParseSet("", fs)
}
func main() {
	// Configure options from environment variables
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		EnvPrefix: "VOTING_",
	})
	failOnError(err, "Failed to parse options")
	conf.ParseAll()

	// Establish RabbitMQ connection
	log.Print("Connecting to AMQP...")
	rabbitConnection, err := amqp.Dial(*rabbitmqConnectionString)
	failOnError(err, "Failed to connect to RabbitMQ")
	defer rabbitConnection.Close()

	// Establish Redis connection
	log.Print("Connecting to Redis...")
	redisConnection, err := redis.DialURL(*redisConnectionString)
	failOnError(err, "Failed to connect to Redis")
	defer redisConnection.Close()

	// Open a channel
	channel, err := rabbitConnection.Channel()
	failOnError(err, "Failed to open a channel")
	defer channel.Close()

	// Declare the queue
	queue, err := channel.QueueDeclare(
		"votes", // Queue name
		true,    // durable
		false,   // delete when unused
		false,   // exclusive
		false,   // no-wait
		nil,     // arguments
	)
	failOnError(err, "Failed to declare a queue")

	// Consume messages off the queue
	msgs, err := channel.Consume(
		queue.Name, // queue name
		"",         // consumer
		false,      // auto-ack
		false,      // exclusive
		false,      // no-local
		false,      // no-wait
		nil,        // args
	)
	failOnError(err, "Failed to register a consumer")

	// Create channel for consuming messages
	consume := make(chan bool)

	go func() {
		for msg := range msgs {
			log.Printf("Received a message: %s", msg.Body)
			var team Team
			err := json.Unmarshal(msg.Body, &team)
			failOnError(err, "Failed to unmarshal JSON body of message")

			// Increment vote count for team
			redisConnection.Do("INCR", fmt.Sprintf("%s", team.Team))
			log.Printf("Incremented vote count for team %s", team.Team)

			msg.Ack(false)
		}
	}()

	log.Printf(" [*] Waiting for messages. To exit press CTRL+C")
	<-consume
}
Exemple #21
0
func getConfig(flagset *flag.FlagSet, userCfgFile string) (*config.Config, error) {
	opts := globalconf.Options{EnvPrefix: "FLEET_"}

	if userCfgFile != "" {
		// Fail hard if a user-provided config is not usable
		fi, err := os.Stat(userCfgFile)
		if err != nil {
			log.Fatalf("Unable to use config file %s: %v", userCfgFile, err)
		}
		if fi.IsDir() {
			log.Fatalf("Provided config %s is a directory, not a file", userCfgFile)
		}

		log.Infof("Using provided config file %s", userCfgFile)
		opts.Filename = userCfgFile

	} else if _, err := os.Stat(DefaultConfigFile); err == nil {
		log.Infof("Using default config file %s", DefaultConfigFile)
		opts.Filename = DefaultConfigFile
	} else {
		log.Infof("No provided or default config file found - proceeding without")
	}

	gconf, err := globalconf.NewWithOptions(&opts)
	if err != nil {
		return nil, err
	}

	gconf.ParseSet("", flagset)

	cfg := config.Config{
		Verbosity:               (*flagset.Lookup("verbosity")).Value.(flag.Getter).Get().(int),
		EtcdServers:             (*flagset.Lookup("etcd_servers")).Value.(flag.Getter).Get().(pkg.StringSlice),
		EtcdUsername:            (*flagset.Lookup("etcd_username")).Value.(flag.Getter).Get().(string),
		EtcdPassword:            (*flagset.Lookup("etcd_password")).Value.(flag.Getter).Get().(string),
		EtcdKeyPrefix:           (*flagset.Lookup("etcd_key_prefix")).Value.(flag.Getter).Get().(string),
		EtcdKeyFile:             (*flagset.Lookup("etcd_keyfile")).Value.(flag.Getter).Get().(string),
		EtcdCertFile:            (*flagset.Lookup("etcd_certfile")).Value.(flag.Getter).Get().(string),
		EtcdCAFile:              (*flagset.Lookup("etcd_cafile")).Value.(flag.Getter).Get().(string),
		EtcdRequestTimeout:      (*flagset.Lookup("etcd_request_timeout")).Value.(flag.Getter).Get().(float64),
		EngineReconcileInterval: (*flagset.Lookup("engine_reconcile_interval")).Value.(flag.Getter).Get().(float64),
		PublicIP:                (*flagset.Lookup("public_ip")).Value.(flag.Getter).Get().(string),
		RawMetadata:             (*flagset.Lookup("metadata")).Value.(flag.Getter).Get().(string),
		AgentTTL:                (*flagset.Lookup("agent_ttl")).Value.(flag.Getter).Get().(string),
		DisableEngine:           (*flagset.Lookup("disable_engine")).Value.(flag.Getter).Get().(bool),
		DisableWatches:          (*flagset.Lookup("disable_watches")).Value.(flag.Getter).Get().(bool),
		EnableGRPC:              (*flagset.Lookup("enable_grpc")).Value.(flag.Getter).Get().(bool),
		VerifyUnits:             (*flagset.Lookup("verify_units")).Value.(flag.Getter).Get().(bool),
		UnitsDirectory:          (*flagset.Lookup("units_directory")).Value.(flag.Getter).Get().(string),
		SystemdUser:             (*flagset.Lookup("systemd_user")).Value.(flag.Getter).Get().(bool),
		TokenLimit:              (*flagset.Lookup("token_limit")).Value.(flag.Getter).Get().(int),
		AuthorizedKeysFile:      (*flagset.Lookup("authorized_keys_file")).Value.(flag.Getter).Get().(string),
	}

	if cfg.VerifyUnits {
		log.Error("Config option verify_units is no longer supported - ignoring")
	}
	if len(cfg.AuthorizedKeysFile) > 0 {
		log.Error("Config option authorized_keys_file is no longer supported - ignoring")
	}

	if cfg.Verbosity > 0 {
		log.EnableDebug()
	}

	return &cfg, nil
}
Exemple #22
0
func main() {
	flag.Parse()
	// Set 'cfile' here if *confFile exists, because we should only try and
	// parse the conf file if it exists. If we try and parse the default
	// conf file location when it's not there, we (unsurprisingly) get a
	// panic.
	var cfile string
	if _, err := os.Stat(*confFile); err == nil {
		cfile = *confFile
	}

	// Still parse globalconf, though, even if the config file doesn't exist
	// because we want to be able to use environment variables.
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename:  cfile,
		EnvPrefix: "RTPROBE_",
	})
	if err != nil {
		panic(fmt.Sprintf("error with configuration file: %s", err))
	}
	conf.ParseAll()

	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
	// workaround for https://github.com/grafana/grafana/issues/4055
	switch *logLevel {
	case 0:
		log.Level(log.TRACE)
	case 1:
		log.Level(log.DEBUG)
	case 2:
		log.Level(log.INFO)
	case 3:
		log.Level(log.WARN)
	case 4:
		log.Level(log.ERROR)
	case 5:
		log.Level(log.CRITICAL)
	case 6:
		log.Level(log.FATAL)
	}

	if *showVersion {
		fmt.Printf("raintank-probe (built with %s, git hash %s)\n", runtime.Version(), GitHash)
		return
	}

	if *nodeName == "" {
		log.Fatal(4, "name must be set.")
	}

	file, err := ioutil.ReadFile(*publicChecksFile)
	if err != nil {
		log.Error(3, "Could not read publicChecks file. %s", err.Error())
	} else {
		err = json.Unmarshal(file, &PublicChecks)
		if err != nil {
			log.Error(3, "Could not parse publicChecks file. %s", err.Error())
		}
	}

	jobScheduler := scheduler.New(*healthHosts)
	go jobScheduler.CheckHealth()

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	controllerUrl, err := url.Parse(*serverAddr)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	controllerUrl.Path = path.Clean(controllerUrl.Path + "/socket.io")
	version := strings.Split(GitHash, "-")[0]
	controllerUrl.RawQuery = fmt.Sprintf("EIO=3&transport=websocket&apiKey=%s&name=%s&version=%s", *apiKey, url.QueryEscape(*nodeName), version)

	if controllerUrl.Scheme != "ws" && controllerUrl.Scheme != "wss" {
		log.Fatal(4, "invalid server address.  scheme must be ws or wss. was %s", controllerUrl.Scheme)
	}

	tsdbUrl, err := url.Parse(*tsdbAddr)
	if err != nil {
		log.Fatal(4, "Invalid TSDB url.", err)
	}
	if !strings.HasPrefix(tsdbUrl.Path, "/") {
		tsdbUrl.Path += "/"
	}
	publisher.Init(tsdbUrl, *apiKey, *concurrency)

	client, err := gosocketio.Dial(controllerUrl.String(), transport.GetDefaultWebsocketTransport())
	if err != nil {
		log.Fatal(4, "unable to connect to server on url %s: %s", controllerUrl.String(), err)
	}
	bindHandlers(client, controllerUrl, jobScheduler, interrupt)

	//wait for interupt Signal.
	<-interrupt
	log.Info("interrupt")
	jobScheduler.Close()
	client.Close()
	return
}
func main() {
	flag.Parse()

	// Only try and parse the conf file if it exists
	if _, err := os.Stat(*confFile); err == nil {
		conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
		if err != nil {
			log.Fatal(4, "error with configuration file: %s", err)
			os.Exit(1)
		}
		conf.ParseAll()
	}

	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))

	if *showVersion {
		fmt.Println("metrics_tank")
		return
	}
	if *instance == "" {
		log.Fatal(0, "instance can't be empty")
	}
	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(0, "failed to lookup hostname. %s", err)
	}
	stats, err := helper.New(true, *statsdAddr, *statsdType, "metric_tank", strings.Replace(hostname, ".", "_", -1))
	if err != nil {
		log.Fatal(0, "failed to initialize statsd. %s", err)
	}

	if *channel == "" {
		rand.Seed(time.Now().UnixNano())
		*channel = fmt.Sprintf("metric_tank%06d#ephemeral", rand.Int()%999999)
	}

	if *topic == "" {
		log.Fatal(0, "--topic is required")
	}

	if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
		log.Fatal(0, "--nsqd-tcp-address or --lookupd-http-address required")
	}
	if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
		log.Fatal(0, "use --nsqd-tcp-address or --lookupd-http-address not both")
	}
	// set default cassandra address if none is set.
	if *cassandraAddrs == "" {
		*cassandraAddrs = "localhost"
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	cfg := nsq.NewConfig()
	cfg.UserAgent = "metrics_tank"
	err = app.ParseOpts(cfg, *consumerOpts)
	if err != nil {
		log.Fatal(0, "failed to parse nsq consumer options. %s", err)
	}
	cfg.MaxInFlight = *maxInFlight

	consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", stats)
	if err != nil {
		log.Fatal(0, "Failed to create NSQ consumer. %s", err)
	}

	initMetrics(stats)

	err = InitCassandra()

	if err != nil {
		log.Fatal(4, "failed to initialize cassandra. %s", err)
	}

	set := strings.Split(*aggSettings, ",")
	finalSettings := make([]aggSetting, 0)
	for _, v := range set {
		if v == "" {
			continue
		}
		fields := strings.Split(v, ":")
		if len(fields) != 3 {
			log.Fatal(0, "bad agg settings")
		}
		aggSpan, err := strconv.Atoi(fields[0])
		if err != nil {
			log.Fatal(0, "bad agg settings", err)
		}
		aggChunkSpan, err := strconv.Atoi(fields[1])
		if err != nil {
			log.Fatal(0, "bad agg settings", err)
		}
		aggNumChunks, err := strconv.Atoi(fields[2])
		if err != nil {
			log.Fatal(0, "bad agg settings", err)
		}
		finalSettings = append(finalSettings, aggSetting{uint32(aggSpan), uint32(aggChunkSpan), uint32(aggNumChunks)})
	}

	metrics = NewAggMetrics(uint32(*chunkSpan), uint32(*numChunks), uint32(*chunkMaxStale), uint32(*metricMaxStale), finalSettings)
	handler := NewHandler(metrics)
	consumer.AddConcurrentHandlers(handler, *concurrency)

	nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
	if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
		nsqdAdds = []string{}
	}
	err = consumer.ConnectToNSQDs(nsqdAdds)
	if err != nil {
		log.Fatal(4, "failed to connect to NSQDs. %s", err)
	}
	log.Info("connected to nsqd")

	lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
	if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
		lookupdAdds = []string{}
	}
	err = consumer.ConnectToNSQLookupds(lookupdAdds)
	if err != nil {
		log.Fatal(4, "failed to connect to NSQLookupds. %s", err)
	}

	go func() {
		m := &runtime.MemStats{}
		for range time.Tick(time.Duration(1) * time.Second) {
			runtime.ReadMemStats(m)
			alloc.Value(int64(m.Alloc))
			totalAlloc.Value(int64(m.TotalAlloc))
			sysBytes.Value(int64(m.Sys))
		}
	}()

	go func() {
		http.HandleFunc("/get", Get)
		log.Info("starting listener for metrics and http/debug on %s", *listenAddr)
		log.Info("%s", http.ListenAndServe(*listenAddr, nil))
	}()

	for {
		select {
		case <-consumer.StopChan:
			err := metrics.Persist()
			if err != nil {
				log.Error(3, "failed to persist aggmetrics. %s", err)
			}
			log.Info("closing cassandra session.")
			cSession.Close()
			log.Info("terminating.")
			log.Close()
			return
		case <-sigChan:
			log.Info("Shutting down")
			consumer.Stop()
		}
	}
}