func ReadApiConfig(fileLocation string) (a *ApiConfig) {
f := flag.NewFlagSet("api", flag.ContinueOnError)
flagApiHostname := f.String("api_host", "127.0.0.1", "Api hostname")
flagApiPort := f.Uint("api_port", 8844, "Api port")
globalconf.Register("api", f)
g, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: fileLocation,
})
if err != nil {
log.Printf("[config][warning] Could not parse config file %s"+
" Error: %s, Falling back to default settings",
fileLocation, err.Error())
} else {
g.ParseAll()
}
return &ApiConfig{
Host: *flagApiHostname,
Port: uint16(*flagApiPort),
}
}
func main() {
// Get the configuration
//
if *configFile == "" {
*configFile = "./nightcrawler.ini"
}
opts := globalconf.Options{Filename: *configFile, EnvPrefix: "NC_"}
conf, _ := globalconf.NewWithOptions(&opts)
conf.ParseAll()
// Static assets should be served by nginx in production
//
println(martini.Env)
m := martini.Classic()
if martini.Env != "production" {
m.Use(martini.Static("assets"))
}
// This is just a temp route in development mode
//
if martini.Env == "development" {
m.Get("/", UserRendererMiddleware(), func(r render.Render) {
r.HTML(200, "root", nil)
})
}
// Top level of app
// This section is everything outside of authentication
//
m.Group("/app", func(r martini.Router) {
r.Get("/sign_in", controllers.SignIn)
r.Post("/sign_in", csrf.Validate, controllers.SignIn)
r.Get("/sign_up", controllers.SignUp)
r.Post("/sign_up", csrf.Validate, controllers.CreateSignUp)
r.Get("/sign_out", controllers.SignOut)
r.NotFound(func(x render.Render) {
x.HTML(404, "404", nil)
})
},
CookieStore(*userCookieSecret, *userSessionName),
UserSessionAuth(),
CSRFMiddleware(*userSessionSecret, *userSessionKey),
UserRendererMiddleware(),
)
// Admin interface
m.Group("/app/admin", func(r martini.Router) {
r.Get("/sign_in", controllers.AdminSignIn)
r.NotFound(func(x render.Render) {
x.HTML(404, "404_admin", nil)
})
},
CookieStore("admin123", "nightcrawler_admin"),
AdminSessionAuth(),
CSRFMiddleware("admin123", "adminId"),
AdminRendererMiddleware(),
)
m.Run()
}
func setup(name string) error {
var (
conf *globalconf.GlobalConf
file string
err error
)
// Check for environment variable override, and return error if override is set but file is
// inaccessible. Otherwise, try for the default global location (in the "/etc" directory).
if file := os.Getenv(strings.ToUpper(name) + "_CONFIG"); file != "" {
if _, err = os.Stat(file); err != nil {
return err
}
} else {
file = "/etc/" + name + "/" + name + ".conf"
if _, err = os.Stat(file); err != nil {
file = ""
}
}
// Load from specific configuration file if set, or use local configuration file as a fallback.
if file != "" {
options := &globalconf.Options{Filename: file, EnvPrefix: ""}
if conf, err = globalconf.NewWithOptions(options); err != nil {
return err
}
} else if conf, err = globalconf.New(name); err != nil {
return err
}
conf.EnvPrefix = strings.ToUpper(name) + "_"
conf.ParseAll()
return nil
}
func ReadDbConfig(fileLocation string) *DatabaseConfig {
f := flag.NewFlagSet("database", flag.ContinueOnError)
flagDBHostname := f.String("db_host", "127.0.0.1", "Database hostname")
flagDBPort := f.Uint("db_port", 5432, "Database port")
flagDBUsername := f.String("db_username", "ubi", "Database username")
flagDBPassword := f.String("db_password", "password", "Database password")
flagDBName := f.String("db_name", "ubi_course", "Database name")
flagDBSSLMode := f.String("db_sslmode", "disable", "Database SSL mode")
globalconf.Register("database", f)
g, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: fileLocation,
})
if err != nil {
log.Printf("[config][warning] Could not parse config file '%s': %s. "+
"Falling back to default settings",
fileLocation, err.Error())
} else {
g.ParseAll()
}
return &DatabaseConfig{
Hostname: *flagDBHostname,
Port: uint16(*flagDBPort),
Username: *flagDBUsername,
Password: *flagDBPassword,
Name: *flagDBName,
SSLMode: *flagDBSSLMode,
}
}
func ReadDaqConfig(fileLocation string) *DaqConfig {
f := flag.NewFlagSet("daq", flag.ContinueOnError)
flagDaqInterval := f.Uint("daq_interval", 5, "Daq interval in minutes")
flagMockerOn := f.Bool("daq_mocker_on", false, "Mocker state")
globalconf.Register("daq", f)
g, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: fileLocation,
})
if err != nil {
log.Printf("[config][warning] Could not parse config file %s"+
" Error: %s, Falling back to default settings",
fileLocation, err.Error())
} else {
g.ParseAll()
}
return &DaqConfig{
Interval: *flagDaqInterval,
MockerOn: *flagMockerOn,
}
}
// parse conf from env and args
func parseConf() {
// let mflag parse first
mflag.Parse()
// using gconf parse env
gconf, err := globalconf.NewWithOptions(&globalconf.Options{
EnvPrefix: "WICKET_",
})
if err != nil {
log.Fatalf("error parsing config file: %v", err)
}
fs := flag.NewFlagSet("", flag.ContinueOnError)
mflag.VisitAll(func(f *mflag.Flag) {
for _, n := range f.Names {
if len(n) < 2 {
continue
}
n = strings.TrimPrefix(n, "-")
fs.Var(f.Value, n, f.Usage)
}
})
gconf.ParseSet("", fs)
}
func ReadConfig(filename string) Config {
dbFlagSet := flag.NewFlagSet("mysql", flag.ExitOnError)
flagUser := dbFlagSet.String("username", "", "Database username")
flagPass := dbFlagSet.String("password", "", "Database password")
flagHost := dbFlagSet.String("host", "localhost", "Database host")
flagDbName := dbFlagSet.String("database", "", "Database name")
globalconf.Register("mysql", dbFlagSet)
conf, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: filename,
})
if err != nil {
log.Fatalf(err.Error())
}
conf.ParseAll()
return Config{
Username: *flagUser,
Host: *flagHost,
Password: *flagPass,
Database: *flagDbName,
}
}
func GetConfig() (*Config, error) {
c := &Config{}
var keyFile string
var keyContents string
var proxyProtoHttpsPorts string
flag.StringVar(&keyFile, "jwt-public-key-file", "", "Location of the public-key used to validate JWTs.")
flag.StringVar(&keyContents, "jwt-public-key-contents", "", "An alternative to jwt-public-key-file. The contents of the key.")
flag.StringVar(&c.ListenAddr, "listen-address", ":8080", "The tcp address to listen on.")
flag.StringVar(&c.CattleAddr, "cattle-address", "", "The tcp address to forward cattle API requests to. Will not proxy to cattle api if this option is not provied.")
flag.IntVar(&c.ParentPid, "parent-pid", 0, "If provided, this process will exit when the specified parent process stops running.")
flag.StringVar(&proxyProtoHttpsPorts, "https-proxy-protocol-ports", "", "If proxy protocol is used, a list of proxy ports that will allow us to recognize that the connection was over https.")
confOptions := &globalconf.Options{
EnvPrefix: "PROXY_",
}
conf, err := globalconf.NewWithOptions(confOptions)
if err != nil {
return nil, err
}
conf.ParseAll()
if keyFile != "" && keyContents != "" {
return nil, fmt.Errorf("Can't specify both jwt-public-key-file and jwt-public-key-contents")
}
var parsedKey interface{}
var parseErr error
if keyFile != "" {
parsedKey, parseErr = ParsePublicKey(keyFile)
} else if keyContents != "" {
parsedKey, parseErr = ParsePublicKeyFromMemory(keyContents)
} else {
parseErr = fmt.Errorf("Must specify one of jwt-public-key-file and jwt-public-key-contents")
}
if parseErr != nil {
return nil, parseErr
}
c.PublicKey = parsedKey
portMap := make(map[int]bool)
ports := strings.Split(proxyProtoHttpsPorts, ",")
for _, port := range ports {
if p, err := strconv.Atoi(port); err == nil {
portMap[p] = true
}
}
c.ProxyProtoHttpsPorts = portMap
return c, nil
}
func Parse() error {
flag.BoolVar(&Config.HaProxyMonitor, "haproxy-monitor", false, "Monitor HAProxy")
flag.IntVar(&Config.Port, "port", 8080, "Listen port")
flag.StringVar(&Config.Ip, "ip", "", "Listen IP, defaults to all IPs")
flag.StringVar(&Config.CAdvisorUrl, "cadvisor-url", "http://localhost:8081", "cAdvisor URL")
flag.StringVar(&Config.DockerUrl, "docker-host", "unix:///var/run/docker.sock", "Docker host URL")
flag.IntVar(&Config.NumStats, "num-stats", 600, "Number of stats to show by default")
flag.BoolVar(&Config.Auth, "auth", false, "Authenticate requests")
flag.StringVar(&Config.HostUuid, "host-uuid", "", "Host UUID")
flag.BoolVar(&Config.HostUuidCheck, "host-uuid-check", true, "Validate host UUID")
flag.StringVar(&Config.Key, "public-key", "", "Public Key for Authentication")
flag.IntVar(&Config.EventsPoolSize, "events-pool-size", 10, "Size of worker pool for processing docker events.")
flag.StringVar(&Config.CattleUrl, "cattle-url", "", "URL for accessing cattle api")
flag.StringVar(&Config.CattleAccessKey, "cattle-access-key", "", "Access key for cattle api")
flag.StringVar(&Config.CattleSecretKey, "cattle-secret-key", "", "Secret key for cattle api")
flag.StringVar(&Config.CattleStateDir, "cattle-state-dir", "", "Directory where Rancher state is persisted.")
flag.StringVar(&Config.PidFile, "pid-file", "", "PID file")
flag.StringVar(&Config.LogFile, "log", "", "Log file")
confOptions := &globalconf.Options{
EnvPrefix: "HOST_API_",
}
filename := os.Getenv("HOST_API_CONFIG_FILE")
if len(filename) > 0 {
confOptions.Filename = filename
}
conf, err := globalconf.NewWithOptions(confOptions)
if err != nil {
return err
}
conf.ParseAll()
if len(Config.Key) > 0 {
if err := ParsedPublicKey(); err != nil {
glog.Error("Error reading file")
return err
}
}
s, err := os.Stat("/run/systemd/system")
if err != nil || !s.IsDir() {
Config.Systemd = false
} else {
Config.Systemd = true
}
return nil
}
// TODO: html template
// TODO: email this out
// TODO: better comments
func main() {
// loading configuration
flag.Parse()
conf, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: config,
})
if err != nil {
log.Fatal(err)
}
conf.ParseAll()
ccAPI, _ := api.NewAPI(config, authToken, apiKey, daysBack, debug)
// Generate a report from a stored file
if prevReport != "" {
c := types.Load(prevReport)
c.BuildCampaignReport(getTopDomains, getTopClicks)
fmt.Println(types.Render(c))
os.Exit(0)
}
runTime := time.Now().Format("2006-01-02T15:04")
saveTo := fmt.Sprintf("./logs/%s.gob", runTime)
log.Print(runTime, saveTo)
camps, err := ccAPI.GetCampaigns()
if err != nil {
log.Fatal(err)
}
log.Print(camps)
log.Print("retrieved campaigns: ", len(camps.Campaigns))
for _, c := range camps.Campaigns {
err = ccAPI.GetCampaignDetail(c)
log.Print(err)
log.Print(c)
err = ccAPI.GetCampaignPreview(c)
log.Print(err)
err = ccAPI.GetCampaignTracking(c)
b, _ := json.MarshalIndent(c, "", " ")
fmt.Println(string(b))
}
types.Save(camps, saveTo)
camps.BuildCampaignReport(getTopDomains, getTopClicks)
fmt.Println(camps)
fmt.Println(types.Render(camps))
}
func init() {
chanmap = make(map[string]*hbot.IrcChannel)
config, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: configFile,
})
if err != nil {
log.Fatalf("Cannot parse configuration file: %s", err)
}
globalconf.Register("tumblr", tumblrConf)
globalconf.Register("twitter", twitterConf)
globalconf.Register("logger", loggerConf)
globalconf.Register("commands", commandsConf)
config.ParseAll()
}
func ConfInit() {
flag.Parse()
if *g_conf_file == "" {
//panic("please set conf file ")
println("have no config file")
return
}
conf, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: *g_conf_file,
})
if err != nil {
panic(err)
}
conf.ParseAll()
}
func getConfig(flagset *flag.FlagSet, userCfgFile string) (*config.Config, error) {
opts := globalconf.Options{EnvPrefix: "API_HOSTD_"}
if userCfgFile != "" {
// Fail hard if a user-provided config is not usable
fi, err := os.Stat(userCfgFile)
if err != nil {
ctxLog.Fatalf("Unable to use config file %s: %v", userCfgFile, err)
}
if fi.IsDir() {
ctxLog.Fatalf("Provided config %s is a directory, not a file", userCfgFile)
}
opts.Filename = userCfgFile
} else if _, err := os.Stat(DefaultConfigFile); err == nil {
opts.Filename = DefaultConfigFile
}
gconf, err := globalconf.NewWithOptions(&opts)
if err != nil {
return nil, err
}
gconf.ParseSet("", flagset)
cfg := config.Config{
Verbosity: (*flagset.Lookup("verbosity")).Value.(flag.Getter).Get().(int),
IP: (*flagset.Lookup("ip")).Value.(flag.Getter).Get().(string),
Port: (*flagset.Lookup("port")).Value.(flag.Getter).Get().(string),
Secret: (*flagset.Lookup("jwt_sign_key")).Value.(flag.Getter).Get().(string),
CORSAllowedOrigins: config.StringToSlice((*flagset.Lookup("cors_allowed_origins")).Value.(flag.Getter).Get().(string)),
CORSAllowedMethods: config.StringToSlice((*flagset.Lookup("cors_allowed_methods")).Value.(flag.Getter).Get().(string)),
CORSAllowedHeaders: config.StringToSlice((*flagset.Lookup("cors_allowed_headers")).Value.(flag.Getter).Get().(string)),
CORSExposedHeaders: config.StringToSlice((*flagset.Lookup("cors_exposed_headers")).Value.(flag.Getter).Get().(string)),
CORSAllowCredentials: (*flagset.Lookup("cors_allow_credentials")).Value.(flag.Getter).Get().(bool),
CORSMaxAge: (*flagset.Lookup("cors_max_age")).Value.(flag.Getter).Get().(int),
CORSOptionsPassThrough: (*flagset.Lookup("cors_options_pass_through")).Value.(flag.Getter).Get().(bool),
CORSDebug: (*flagset.Lookup("cors_debug")).Value.(flag.Getter).Get().(bool),
}
log.SetLevel(log.Level(cfg.Verbosity))
ctxLog.Infof("Loaded config: [%+v]", cfg)
return &cfg, nil
}
// Read config
//
// Initialize Config from Config File
func ReadConfig(ConfigFile string, Datadir string, EnvPrefix string) *ConfigManager {
if !FileExist(ConfigFile) {
// create ConfigFile if it does not exist, otherwise
// globalconf will panic when trying to persist flags.
fmt.Printf("config file '%s' doesn't exist, creating it\n", ConfigFile)
os.Create(ConfigFile)
}
g, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: ConfigFile,
EnvPrefix: EnvPrefix,
})
if err != nil {
fmt.Println(err)
} else {
g.ParseAll()
}
cfg := &ConfigManager{ExecPath: Datadir, Debug: true, conf: g, Paranoia: true}
return cfg
}
func GetConfig() (*Config, error) {
c := &Config{}
var keyFile string
var keyContents string
flag.StringVar(&keyFile, "jwt-public-key-file", "", "Location of the public-key used to validate JWTs.")
flag.StringVar(&keyContents, "jwt-public-key-contents", "", "An alternative to jwt-public-key-file. The contents of the key.")
flag.StringVar(&c.ListenAddr, "listen-address", "localhost:8080", "The tcp address to listen on.")
flag.StringVar(&c.CattleAddr, "cattle-address", "", "The tcp address to forward cattle API requests to. Will not proxy to cattle api if this option is not provied.")
flag.IntVar(&c.ParentPid, "parent-pid", 0, "If provided, this process will exit when the specified parent process stops running.")
confOptions := &globalconf.Options{
EnvPrefix: "PROXY_",
}
conf, err := globalconf.NewWithOptions(confOptions)
if err != nil {
return nil, err
}
conf.ParseAll()
if keyFile != "" && keyContents != "" {
return nil, fmt.Errorf("Can't specify both jwt-public-key-file and jwt-public-key-contents")
}
var parsedKey interface{}
var parseErr error
if keyFile != "" {
parsedKey, parseErr = ParsePublicKey(keyFile)
} else if keyContents != "" {
parsedKey, parseErr = ParsePublicKeyFromMemory(keyContents)
} else {
parseErr = fmt.Errorf("Must specify one of jwt-public-key-file and jwt-public-key-contents")
}
if parseErr != nil {
return nil, parseErr
}
c.PublicKey = parsedKey
return c, nil
}
func ReadMockerConfig(fileLocation string) *MockerConfig {
f := flag.NewFlagSet("mocker", flag.ContinueOnError)
flagMockerAmount := f.Uint("mocker_place_amount", 5, "Amount of mocked places")
flagMockerRecordsMin := f.Uint("mocker_records_min", 0, "Minimum amount of queuers")
flagMockerRecordsMax := f.Uint("mocker_records_max", 5, "Maximum amount of queuers")
flagMockerTimeMin := f.Uint("mocker_time_min", 1, "Minimum time in queue")
flagMockerTimeMax := f.Uint("mocker_time_max", 50, "Maximum time in queue")
flagMockerHistoric := f.Bool("mocker_historic", false, "Enable historic data generation")
flagMockerHistoricWeeks := f.Uint("mocker_historic_weeks", 0, "Number of weeks to generate historic data")
flagMockerHistoricInterval := f.Uint("mocker_historic_interval", 15, "Interval for gistoric records")
globalconf.Register("mocker", f)
g, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: fileLocation,
})
if err != nil {
log.Printf("[config][warning] Could not parse config file %s"+
" Error: %s, Falling back to default settings",
fileLocation, err.Error())
} else {
g.ParseAll()
}
return &MockerConfig{
PlaceAmount: *flagMockerAmount,
RecordsMin: *flagMockerRecordsMin,
RecordsMax: *flagMockerRecordsMax,
TimeMin: *flagMockerTimeMin,
TimeMax: *flagMockerTimeMax,
Historic: *flagMockerHistoric,
HistoricWeek: *flagMockerHistoricWeeks,
HistoricInterval: *flagMockerHistoricInterval,
}
}
func main() {
flag.Parse()
// Only try and parse the conf file if it exists
if _, err := os.Stat(*confFile); err == nil {
conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
if err != nil {
log.Fatal(4, err.Error())
}
conf.ParseAll()
}
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
if *showVersion {
fmt.Println("nsq_probe_events_to_elasticsearch")
return
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatal(4, "--topic is required")
}
if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
log.Fatal(4, "--nsqd-tcp-address or --lookupd-http-address required")
}
if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
log.Fatal(4, "use --nsqd-tcp-address or --lookupd-http-address not both")
}
hostname, err := os.Hostname()
if err != nil {
log.Fatal(4, err.Error())
}
metrics, err := helper.New(true, *statsdAddr, *statsdType, "nsq_probe_events_to_elasticsearch", strings.Replace(hostname, ".", "_", -1))
if err != nil {
log.Fatal(4, err.Error())
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
eventsToEsOK = metrics.NewCount("events_to_es.ok")
eventsToEsFail = metrics.NewCount("events_to_es.fail")
messagesSize = metrics.NewMeter("message_size", 0)
msgsAge = metrics.NewMeter("message_age", 0)
esPutDuration = metrics.NewTimer("es_put_duration", 0)
msgsHandleOK = metrics.NewCount("handle.ok")
msgsHandleFail = metrics.NewCount("handle.fail")
cfg := nsq.NewConfig()
cfg.UserAgent = "nsq_probe_events_to_elasticsearch"
err = app.ParseOpts(cfg, *consumerOpts)
if err != nil {
log.Fatal(4, err.Error())
}
cfg.MaxInFlight = *maxInFlight
consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", metrics)
if err != nil {
log.Fatal(4, err.Error())
}
handler, err := NewESHandler()
if err != nil {
log.Fatal(4, err.Error())
}
consumer.AddConcurrentHandlers(handler, 80)
nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
nsqdAdds = []string{}
}
err = consumer.ConnectToNSQDs(nsqdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
log.Info("connected to nsqd")
lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
lookupdAdds = []string{}
}
err = consumer.ConnectToNSQLookupds(lookupdAdds)
if err != nil {
log.Fatal(4, err.Error())
}
go func() {
log.Info("INFO starting listener for http/debug on %s", *listenAddr)
httperr := http.ListenAndServe(*listenAddr, nil)
if httperr != nil {
log.Info(httperr.Error())
}
}()
for {
select {
case <-consumer.StopChan:
return
case <-sigChan:
consumer.Stop()
}
}
}
func main() {
flag.Parse()
// Set 'cfile' here if *confFile exists, because we should only try and
// parse the conf file if it exists. If we try and parse the default
// conf file location when it's not there, we (unsurprisingly) get a
// panic.
var cfile string
if _, err := os.Stat(*confFile); err == nil {
cfile = *confFile
}
// Still parse globalconf, though, even if the config file doesn't exist
// because we want to be able to use environment variables.
conf, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: cfile,
EnvPrefix: "TASKAGENT_",
})
if err != nil {
panic(fmt.Sprintf("error with configuration file: %s", err))
}
conf.ParseAll()
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
// workaround for https://github.com/grafana/grafana/issues/4055
switch *logLevel {
case 0:
log.Level(log.TRACE)
case 1:
log.Level(log.DEBUG)
case 2:
log.Level(log.INFO)
case 3:
log.Level(log.WARN)
case 4:
log.Level(log.ERROR)
case 5:
log.Level(log.CRITICAL)
case 6:
log.Level(log.FATAL)
}
if *showVersion {
fmt.Printf("task-agent (built with %s, git hash %s)\n", runtime.Version(), GitHash)
return
}
if *nodeName == "" {
log.Fatal(4, "name must be set.")
}
snapUrl, err := url.Parse(*snapUrlStr)
if err != nil {
log.Fatal(4, "could not parse snapUrl. %s", err)
}
snapClient, err := snap.NewClient(*nodeName, *tsdbAddr, *apiKey, snapUrl)
if err != nil {
log.Fatal(4, err.Error())
}
InitTaskCache(snapClient)
interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, os.Interrupt)
shutdownStart := make(chan struct{})
controllerUrl, err := url.Parse(*serverAddr)
if err != nil {
log.Fatal(4, err.Error())
}
controllerUrl.Path = path.Clean(controllerUrl.Path + fmt.Sprintf("/socket/%s/%d", *nodeName, Version))
if controllerUrl.Scheme != "ws" && controllerUrl.Scheme != "wss" {
log.Fatal(4, "invalid server address. scheme must be ws or wss. was %s", controllerUrl.Scheme)
}
conn, err := connect(controllerUrl)
if err != nil {
log.Fatal(4, "unable to connect to server on url %s: %s", controllerUrl.String(), err)
}
//create new session, allow 1000 events to be queued in the writeQueue before Emit() blocks.
sess := session.NewSession(conn, 1000)
sess.On("disconnect", func() {
// on disconnect, reconnect.
ticker := time.NewTicker(time.Second)
connected := false
for !connected {
select {
case <-shutdownStart:
ticker.Stop()
return
case <-ticker.C:
conn, err := connect(controllerUrl)
if err == nil {
sess.Conn = conn
connected = true
go sess.Start()
}
}
}
ticker.Stop()
})
sess.On("heartbeat", func(body []byte) {
log.Debug("recieved heartbeat event. %s", body)
})
sess.On("taskList", HandleTaskList())
sess.On("taskUpdate", HandleTaskUpdate())
sess.On("taskAdd", HandleTaskAdd())
sess.On("taskRemove", HandleTaskRemove())
go sess.Start()
//periodically send an Updated Catalog.
go SendCatalog(sess, snapClient, shutdownStart)
// connect to the snap server and monitor that it is up.
go snapClient.Run()
//wait for interupt Signal.
<-interrupt
log.Info("interrupt")
close(shutdownStart)
sess.Close()
return
}
func initConfig() {
configfile := mflag.String([]string{"-config"}, "/etc/sshpiperd.conf", "Config file path. Note: any option will be overwrite if it is set by commandline")
mflag.StringVar(&config.ListenAddr, []string{"l", "-listen_addr"}, "0.0.0.0", "Listening Address")
mflag.UintVar(&config.Port, []string{"p", "-port"}, 2222, "Listening Port")
mflag.StringVar(&config.WorkingDir, []string{"w", "-working_dir"}, "/var/sshpiper", "Working Dir")
mflag.StringVar(&config.PiperKeyFile, []string{"i", "-server_key"}, "/etc/ssh/ssh_host_rsa_key", "Key file for SSH Piper")
mflag.StringVar(&config.Challenger, []string{"c", "-challenger"}, "", "Additional challenger name, e.g. pam, emtpy for no additional challenge")
mflag.StringVar(&config.Logfile, []string{"-log"}, "", "Logfile path. Leave emtpy or any error occurs will fall back to stdout")
mflag.BoolVar(&config.AllowBadUsername, []string{"-allow_bad_username"}, false, "disable username check while search the working dir")
mflag.BoolVar(&config.ShowHelp, []string{"h", "-help"}, false, "Print help and exit")
mflag.BoolVar(&config.ShowVersion, []string{"-version"}, false, "Print version and exit")
mflag.Parse()
if _, err := os.Stat(*configfile); os.IsNotExist(err) {
if !mflag.IsSet("-config") {
*configfile = ""
} else {
logger.Fatalf("config file %v not found", *configfile)
}
}
gconf, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: *configfile,
EnvPrefix: "SSHPIPERD_",
})
if err != nil { // this error will happen only if file error
logger.Fatalln("load config file error %v: %v", *configfile, err)
}
// build a dummy flag set for globalconf to parse
fs := flag.NewFlagSet("", flag.ContinueOnError)
ignoreSet := make(map[string]bool)
mflag.Visit(func(f *mflag.Flag) {
for _, n := range f.Names {
ignoreSet[n] = true
}
})
// should be ignored
ignoreSet["-help"] = true
ignoreSet["-version"] = true
mflag.VisitAll(func(f *mflag.Flag) {
for _, n := range f.Names {
if len(n) < 2 {
continue
}
if !ignoreSet[n] {
n = strings.TrimPrefix(n, "-")
fs.Var(f.Value, n, f.Usage)
}
}
})
gconf.ParseSet("", fs)
}
func main() {
// Configure options from environment variables
conf, err := globalconf.NewWithOptions(&globalconf.Options{
EnvPrefix: "VOTING_",
})
failOnError(err, "Failed to parse options")
conf.ParseAll()
// Establish RabbitMQ connection
log.Print("Connecting to AMQP...")
rabbitConnection, err := amqp.Dial(*rabbitmqConnectionString)
failOnError(err, "Failed to connect to RabbitMQ")
defer rabbitConnection.Close()
// Establish Redis connection
log.Print("Connecting to Redis...")
redisConnection, err := redis.DialURL(*redisConnectionString)
failOnError(err, "Failed to connect to Redis")
defer redisConnection.Close()
// Open a channel
channel, err := rabbitConnection.Channel()
failOnError(err, "Failed to open a channel")
defer channel.Close()
// Declare the queue
queue, err := channel.QueueDeclare(
"votes", // Queue name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
// Consume messages off the queue
msgs, err := channel.Consume(
queue.Name, // queue name
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
// Create channel for consuming messages
consume := make(chan bool)
go func() {
for msg := range msgs {
log.Printf("Received a message: %s", msg.Body)
var team Team
err := json.Unmarshal(msg.Body, &team)
failOnError(err, "Failed to unmarshal JSON body of message")
// Increment vote count for team
redisConnection.Do("INCR", fmt.Sprintf("%s", team.Team))
log.Printf("Incremented vote count for team %s", team.Team)
msg.Ack(false)
}
}()
log.Printf(" [*] Waiting for messages. To exit press CTRL+C")
<-consume
}
func getConfig(flagset *flag.FlagSet, userCfgFile string) (*config.Config, error) {
opts := globalconf.Options{EnvPrefix: "FLEET_"}
if userCfgFile != "" {
// Fail hard if a user-provided config is not usable
fi, err := os.Stat(userCfgFile)
if err != nil {
log.Fatalf("Unable to use config file %s: %v", userCfgFile, err)
}
if fi.IsDir() {
log.Fatalf("Provided config %s is a directory, not a file", userCfgFile)
}
log.Infof("Using provided config file %s", userCfgFile)
opts.Filename = userCfgFile
} else if _, err := os.Stat(DefaultConfigFile); err == nil {
log.Infof("Using default config file %s", DefaultConfigFile)
opts.Filename = DefaultConfigFile
} else {
log.Infof("No provided or default config file found - proceeding without")
}
gconf, err := globalconf.NewWithOptions(&opts)
if err != nil {
return nil, err
}
gconf.ParseSet("", flagset)
cfg := config.Config{
Verbosity: (*flagset.Lookup("verbosity")).Value.(flag.Getter).Get().(int),
EtcdServers: (*flagset.Lookup("etcd_servers")).Value.(flag.Getter).Get().(pkg.StringSlice),
EtcdUsername: (*flagset.Lookup("etcd_username")).Value.(flag.Getter).Get().(string),
EtcdPassword: (*flagset.Lookup("etcd_password")).Value.(flag.Getter).Get().(string),
EtcdKeyPrefix: (*flagset.Lookup("etcd_key_prefix")).Value.(flag.Getter).Get().(string),
EtcdKeyFile: (*flagset.Lookup("etcd_keyfile")).Value.(flag.Getter).Get().(string),
EtcdCertFile: (*flagset.Lookup("etcd_certfile")).Value.(flag.Getter).Get().(string),
EtcdCAFile: (*flagset.Lookup("etcd_cafile")).Value.(flag.Getter).Get().(string),
EtcdRequestTimeout: (*flagset.Lookup("etcd_request_timeout")).Value.(flag.Getter).Get().(float64),
EngineReconcileInterval: (*flagset.Lookup("engine_reconcile_interval")).Value.(flag.Getter).Get().(float64),
PublicIP: (*flagset.Lookup("public_ip")).Value.(flag.Getter).Get().(string),
RawMetadata: (*flagset.Lookup("metadata")).Value.(flag.Getter).Get().(string),
AgentTTL: (*flagset.Lookup("agent_ttl")).Value.(flag.Getter).Get().(string),
DisableEngine: (*flagset.Lookup("disable_engine")).Value.(flag.Getter).Get().(bool),
DisableWatches: (*flagset.Lookup("disable_watches")).Value.(flag.Getter).Get().(bool),
EnableGRPC: (*flagset.Lookup("enable_grpc")).Value.(flag.Getter).Get().(bool),
VerifyUnits: (*flagset.Lookup("verify_units")).Value.(flag.Getter).Get().(bool),
UnitsDirectory: (*flagset.Lookup("units_directory")).Value.(flag.Getter).Get().(string),
SystemdUser: (*flagset.Lookup("systemd_user")).Value.(flag.Getter).Get().(bool),
TokenLimit: (*flagset.Lookup("token_limit")).Value.(flag.Getter).Get().(int),
AuthorizedKeysFile: (*flagset.Lookup("authorized_keys_file")).Value.(flag.Getter).Get().(string),
}
if cfg.VerifyUnits {
log.Error("Config option verify_units is no longer supported - ignoring")
}
if len(cfg.AuthorizedKeysFile) > 0 {
log.Error("Config option authorized_keys_file is no longer supported - ignoring")
}
if cfg.Verbosity > 0 {
log.EnableDebug()
}
return &cfg, nil
}
func main() {
flag.Parse()
// Set 'cfile' here if *confFile exists, because we should only try and
// parse the conf file if it exists. If we try and parse the default
// conf file location when it's not there, we (unsurprisingly) get a
// panic.
var cfile string
if _, err := os.Stat(*confFile); err == nil {
cfile = *confFile
}
// Still parse globalconf, though, even if the config file doesn't exist
// because we want to be able to use environment variables.
conf, err := globalconf.NewWithOptions(&globalconf.Options{
Filename: cfile,
EnvPrefix: "RTPROBE_",
})
if err != nil {
panic(fmt.Sprintf("error with configuration file: %s", err))
}
conf.ParseAll()
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
// workaround for https://github.com/grafana/grafana/issues/4055
switch *logLevel {
case 0:
log.Level(log.TRACE)
case 1:
log.Level(log.DEBUG)
case 2:
log.Level(log.INFO)
case 3:
log.Level(log.WARN)
case 4:
log.Level(log.ERROR)
case 5:
log.Level(log.CRITICAL)
case 6:
log.Level(log.FATAL)
}
if *showVersion {
fmt.Printf("raintank-probe (built with %s, git hash %s)\n", runtime.Version(), GitHash)
return
}
if *nodeName == "" {
log.Fatal(4, "name must be set.")
}
file, err := ioutil.ReadFile(*publicChecksFile)
if err != nil {
log.Error(3, "Could not read publicChecks file. %s", err.Error())
} else {
err = json.Unmarshal(file, &PublicChecks)
if err != nil {
log.Error(3, "Could not parse publicChecks file. %s", err.Error())
}
}
jobScheduler := scheduler.New(*healthHosts)
go jobScheduler.CheckHealth()
interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, os.Interrupt)
controllerUrl, err := url.Parse(*serverAddr)
if err != nil {
log.Fatal(4, err.Error())
}
controllerUrl.Path = path.Clean(controllerUrl.Path + "/socket.io")
version := strings.Split(GitHash, "-")[0]
controllerUrl.RawQuery = fmt.Sprintf("EIO=3&transport=websocket&apiKey=%s&name=%s&version=%s", *apiKey, url.QueryEscape(*nodeName), version)
if controllerUrl.Scheme != "ws" && controllerUrl.Scheme != "wss" {
log.Fatal(4, "invalid server address. scheme must be ws or wss. was %s", controllerUrl.Scheme)
}
tsdbUrl, err := url.Parse(*tsdbAddr)
if err != nil {
log.Fatal(4, "Invalid TSDB url.", err)
}
if !strings.HasPrefix(tsdbUrl.Path, "/") {
tsdbUrl.Path += "/"
}
publisher.Init(tsdbUrl, *apiKey, *concurrency)
client, err := gosocketio.Dial(controllerUrl.String(), transport.GetDefaultWebsocketTransport())
if err != nil {
log.Fatal(4, "unable to connect to server on url %s: %s", controllerUrl.String(), err)
}
bindHandlers(client, controllerUrl, jobScheduler, interrupt)
//wait for interupt Signal.
<-interrupt
log.Info("interrupt")
jobScheduler.Close()
client.Close()
return
}
func main() {
flag.Parse()
// Only try and parse the conf file if it exists
if _, err := os.Stat(*confFile); err == nil {
conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
if err != nil {
log.Fatal(4, "error with configuration file: %s", err)
os.Exit(1)
}
conf.ParseAll()
}
log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))
if *showVersion {
fmt.Println("metrics_tank")
return
}
if *instance == "" {
log.Fatal(0, "instance can't be empty")
}
hostname, err := os.Hostname()
if err != nil {
log.Fatal(0, "failed to lookup hostname. %s", err)
}
stats, err := helper.New(true, *statsdAddr, *statsdType, "metric_tank", strings.Replace(hostname, ".", "_", -1))
if err != nil {
log.Fatal(0, "failed to initialize statsd. %s", err)
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("metric_tank%06d#ephemeral", rand.Int()%999999)
}
if *topic == "" {
log.Fatal(0, "--topic is required")
}
if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
log.Fatal(0, "--nsqd-tcp-address or --lookupd-http-address required")
}
if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
log.Fatal(0, "use --nsqd-tcp-address or --lookupd-http-address not both")
}
// set default cassandra address if none is set.
if *cassandraAddrs == "" {
*cassandraAddrs = "localhost"
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
cfg := nsq.NewConfig()
cfg.UserAgent = "metrics_tank"
err = app.ParseOpts(cfg, *consumerOpts)
if err != nil {
log.Fatal(0, "failed to parse nsq consumer options. %s", err)
}
cfg.MaxInFlight = *maxInFlight
consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", stats)
if err != nil {
log.Fatal(0, "Failed to create NSQ consumer. %s", err)
}
initMetrics(stats)
err = InitCassandra()
if err != nil {
log.Fatal(4, "failed to initialize cassandra. %s", err)
}
set := strings.Split(*aggSettings, ",")
finalSettings := make([]aggSetting, 0)
for _, v := range set {
if v == "" {
continue
}
fields := strings.Split(v, ":")
if len(fields) != 3 {
log.Fatal(0, "bad agg settings")
}
aggSpan, err := strconv.Atoi(fields[0])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
aggChunkSpan, err := strconv.Atoi(fields[1])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
aggNumChunks, err := strconv.Atoi(fields[2])
if err != nil {
log.Fatal(0, "bad agg settings", err)
}
finalSettings = append(finalSettings, aggSetting{uint32(aggSpan), uint32(aggChunkSpan), uint32(aggNumChunks)})
}
metrics = NewAggMetrics(uint32(*chunkSpan), uint32(*numChunks), uint32(*chunkMaxStale), uint32(*metricMaxStale), finalSettings)
handler := NewHandler(metrics)
consumer.AddConcurrentHandlers(handler, *concurrency)
nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
nsqdAdds = []string{}
}
err = consumer.ConnectToNSQDs(nsqdAdds)
if err != nil {
log.Fatal(4, "failed to connect to NSQDs. %s", err)
}
log.Info("connected to nsqd")
lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
lookupdAdds = []string{}
}
err = consumer.ConnectToNSQLookupds(lookupdAdds)
if err != nil {
log.Fatal(4, "failed to connect to NSQLookupds. %s", err)
}
go func() {
m := &runtime.MemStats{}
for range time.Tick(time.Duration(1) * time.Second) {
runtime.ReadMemStats(m)
alloc.Value(int64(m.Alloc))
totalAlloc.Value(int64(m.TotalAlloc))
sysBytes.Value(int64(m.Sys))
}
}()
go func() {
http.HandleFunc("/get", Get)
log.Info("starting listener for metrics and http/debug on %s", *listenAddr)
log.Info("%s", http.ListenAndServe(*listenAddr, nil))
}()
for {
select {
case <-consumer.StopChan:
err := metrics.Persist()
if err != nil {
log.Error(3, "failed to persist aggmetrics. %s", err)
}
log.Info("closing cassandra session.")
cSession.Close()
log.Info("terminating.")
log.Close()
return
case <-sigChan:
log.Info("Shutting down")
consumer.Stop()
}
}
}