func (this *ConfigMongodb) LoadConfig(cf *conf.Conf) {
this.ShardBaseNum = cf.Int("shard_base_num", 100000)
this.DebugProtocol = cf.Bool("debug_protocol", false)
this.DebugHeartbeat = cf.Bool("debug_heartbeat", false)
this.ShardStrategy = cf.String("shard_strategy", "legacy")
this.ConnectTimeout = cf.Duration("connect_timeout", 4*time.Second)
this.IoTimeout = cf.Duration("io_timeout", 30*time.Second)
this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 2)
this.MaxConnsPerServer = cf.Int("max_conns_per_server",
this.MaxIdleConnsPerServer*5)
this.HeartbeatInterval = cf.Int("heartbeat_interval", 120)
section, err := cf.Section("breaker")
if err == nil {
this.Breaker.loadConfig(section)
}
this.Servers = make(map[string]*ConfigMongodbServer)
for i := 0; i < len(cf.List("servers", nil)); i++ {
section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
if err != nil {
panic(err)
}
server := new(ConfigMongodbServer)
server.ShardBaseNum = this.ShardBaseNum
server.loadConfig(section)
this.Servers[server.Pool] = server
}
log.Debug("mongodb conf: %+v", *this)
}
func (this *SkyOutput) Init(config *conf.Conf) {
const TYPE_SEP = ":"
this.uidFieldType, this.actionFieldType = als.KEY_TYPE_INT, als.KEY_TYPE_STRING
this.uidField = config.String("uid_field", "_log_info.uid")
if strings.Contains(this.uidField, TYPE_SEP) {
p := strings.SplitN(this.uidField, TYPE_SEP, 2)
this.uidField, this.uidFieldType = p[0], p[1]
}
this.actionField = config.String("action_field", "action")
if this.actionField == "" {
panic("empty action field")
}
if strings.Contains(this.actionField, TYPE_SEP) {
p := strings.SplitN(this.actionField, TYPE_SEP, 2)
this.actionField, this.actionFieldType = p[0], p[1]
}
this.project = config.String("project", "")
var (
host string = config.String("host", "localhost")
port int = config.Int("port", 8585)
)
client := sky.NewClient(host)
client.Port = port
if !client.Ping() {
panic(fmt.Sprintf("sky server not running: %s:%d", host, port))
}
this.table, _ = client.GetTable(config.String("table", ""))
if this.table == nil {
panic("must create table in advance")
}
}
func (this *SyslogngInput) Init(config *conf.Conf) {
this.ident = config.String("ident", "")
if this.ident == "" {
panic("empty ident")
}
this.addr = config.String("addr", ":9787")
}
func (this *esConverter) load(section *conf.Conf) {
this.keys = section.StringList("keys", nil)
this.typ = section.String("type", "")
this.currency = section.String("currency", "")
this.rang = section.IntList("range", nil)
this.normalizers = section.StringList("normalizers", nil)
}
func (this *SelfSysInput) Init(config *conf.Conf) {
this.stopChan = make(chan bool)
this.ident = config.String("ident", "")
if this.ident == "" {
panic("empty ident")
}
}
func (this *CardinalityOutput) Init(config *conf.Conf) {
this.checkpoint = config.String("checkpoint", "")
this.counters = stats.NewCardinalityCounter()
if this.checkpoint != "" {
this.counters.Load(this.checkpoint)
}
}
func (this *ConfigMysqlServer) loadConfig(section *conf.Conf) {
this.Pool = section.String("pool", "")
this.Host = section.String("host", "")
this.Port = section.String("port", "3306")
this.DbName = section.String("db", "")
this.User = section.String("username", "")
this.Pass = section.String("password", "")
this.Charset = section.String("charset", "utf8")
if this.Host == "" ||
this.Port == "" ||
this.Pool == "" ||
this.DbName == "" {
panic("required field missing")
}
this.dsn = ""
if this.User != "" {
this.dsn = this.User + ":"
if this.Pass != "" {
this.dsn += this.Pass
}
}
this.dsn += fmt.Sprintf("@tcp(%s:%s)/%s?", this.Host, this.Port, this.DbName)
if this.Charset != "" {
this.dsn += "charset=" + this.Charset
}
if this.conf.Timeout > 0 {
this.dsn += "&timeout=" + this.conf.Timeout.String()
}
}
func (this *ConfigMemcache) LoadConfig(cf *conf.Conf) {
this.Servers = make(map[string]*ConfigMemcacheServer)
this.HashStrategy = cf.String("hash_strategy", "standard")
this.Timeout = cf.Duration("timeout", 4*time.Second)
this.ReplicaN = cf.Int("replica_num", 1)
section, err := cf.Section("breaker")
if err == nil {
this.Breaker.loadConfig(section)
}
this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 3)
this.MaxConnsPerServer = cf.Int("max_conns_per_server",
this.MaxIdleConnsPerServer*10)
for i := 0; i < len(cf.List("servers", nil)); i++ {
section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
if err != nil {
panic(err)
}
server := new(ConfigMemcacheServer)
server.loadConfig(section)
this.Servers[server.Address()] = server
}
log.Debug("memcache conf: %+v", *this)
}
func (this *ConfigMongodbServer) loadConfig(section *conf.Conf) {
this.Pool = section.String("pool", "")
this.Host = section.String("host", "")
this.Port = section.String("port", "27017")
this.DbName = section.String("db", "")
this.ShardBaseNum = section.Int("shard_base_num", this.ShardBaseNum)
this.User = section.String("user", "")
this.Pass = section.String("pass", "")
this.ReplicaSet = section.String("replicaSet", "")
if this.Host == "" ||
this.Port == "" ||
this.Pool == "" ||
this.DbName == "" {
panic("required field missing")
}
// http://docs.mongodb.org/manual/reference/connection-string/
this.uri = "mongodb://" + this.Host + ":" + this.Port + "/"
if this.DbName != "" {
this.uri += this.DbName + "/"
}
if this.ReplicaSet != "" {
this.uri += "?replicaSet=" + this.ReplicaSet
}
}
func (this *FlashlogInput) Init(config *conf.Conf) {
this.dsn = config.String("dsn",
"flashlog:flashlog@unix(/var/run/mysqld/mysqld.sock)/flashlog?charset=utf8")
this.ident = config.String("ident", "")
if this.ident == "" {
panic("empty ident")
}
}
func (this *ConfigRedisServer) loadConfig(cf *conf.Conf) {
this.Addr = cf.String("addr", "")
if this.Addr == "" {
panic("Empty redis server addr")
}
this.MaxIdle = cf.Int("max_idle", 10)
this.MaxActive = cf.Int("max_active", this.MaxIdle*2)
this.IdleTimeout = cf.Duration("idle_timeout", 10*time.Minute)
}
func (this *apiConfig) loadConfig(section *conf.Conf) {
this.controller = section.String("controller", "")
this.action = section.String("action", "")
io, err := section.Section("io")
if err != nil {
panic(err)
}
this.input = io.Object("input", nil)
this.output = io.Object("output", nil)
}
func (this *ConfigMemcacheServer) loadConfig(section *conf.Conf) {
this.host = section.String("host", "")
if this.host == "" {
panic("Empty memcache server host")
}
this.hort = section.String("port", "")
if this.hort == "" {
panic("Empty memcache server port")
}
log.Debug("memcache server: %+v", *this)
}
func (this *configRpc) loadConfig(section *conf.Conf) {
this.listenAddr = section.String("listen_addr", "")
if this.listenAddr == "" {
panic("Empty listen_addr")
}
this.clientTimeout = time.Duration(section.Int("client_timeout", 0)) * time.Second
this.framed = section.Bool("framed", false)
this.protocol = section.String("protocol", "binary")
log.Debug("rpc: %+v", *this)
}
func (this *EsOutput) Init(config *conf.Conf) {
this.stopChan = make(chan bool)
api.Domain = config.String("domain", "localhost")
this.counters = sortedmap.NewSortedMap()
api.Port = config.String("port", "9200")
this.reportInterval = time.Duration(config.Int("report_interval", 30)) * time.Second
this.showProgress = config.Bool("show_progress", true)
this.flushInterval = time.Duration(config.Int("flush_interval", 30)) * time.Second
this.bulkMaxConn = config.Int("bulk_max_conn", 20)
this.bulkMaxDocs = config.Int("bulk_max_docs", 100)
this.bulkMaxBuffer = config.Int("bulk_max_buffer", 10<<20) // 10 MB
this.dryRun = config.Bool("dryrun", false)
}
func (this *CardinalityFilter) Init(config *conf.Conf) {
this.ident = config.String("ident", "")
if this.ident == "" {
panic("empty ident")
}
for i := 0; i < len(config.List("converts", nil)); i++ {
section, err := config.Section(fmt.Sprintf("%s[%d]", "converts", i))
if err != nil {
panic(err)
}
c := cardinalityConverter{}
c.load(section)
this.converters = append(this.converters, c)
}
}
func (this *ConfigMemcache) loadConfig(cf *conf.Conf) {
this.Servers = make(map[string]*ConfigMemcacheServer)
this.HashStrategy = cf.String("hash_strategy", "standard")
for i := 0; i < len(cf.List("servers", nil)); i++ {
section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
if err != nil {
panic(err)
}
server := new(ConfigMemcacheServer)
server.loadConfig(section)
this.Servers[server.Address()] = server
}
log.Debug("memcache: %+v", *this)
}
func loadConfig(cf *conf.Conf) {
config.etcServers = cf.StringList("etcd_servers", nil)
config.faeTemplateFile = cf.String("fae_template_file", "")
config.faeTargetFile = cf.String("fae_target_file", "")
config.maintainTemplateFile = cf.String("maintain_template_file", "")
config.maintainTargetFile = cf.String("maintain_target_file", "")
log.Debug("config: %+v", config)
}
func (this *ConfigRpc) LoadConfig(section *conf.Conf) {
this.ListenAddr = section.String("listen_addr", "")
if this.ListenAddr == "" {
panic("Empty listen_addr")
}
this.SessionTimeout = section.Duration("session_timeout", 30*time.Second)
this.IoTimeout = section.Duration("io_timeout", 2*time.Second)
this.StatsOutputInterval = section.Duration("stats_output_interval", 10*time.Second)
this.Framed = section.Bool("framed", false)
this.BufferSize = section.Int("buffer_size", 4<<10)
this.Protocol = section.String("protocol", "binary")
this.PreforkMode = section.Bool("prefork_mode", false)
this.MaxOutstandingSessions = section.Int("max_outstanding_sessions", 20000)
this.HostMaxCallPerMinute = section.Int("host_max_call_per_minute", 100*60)
log.Debug("rpc conf: %+v", *this)
}
func (this *ConfigMongodbServer) loadConfig(section *conf.Conf) {
this.Kind = section.String("kind", "")
this.Host = section.String("host", "")
this.Port = section.String("port", "27017")
this.DbName = section.String("db", "")
this.ShardBaseNum = section.Int("shard_base_num", this.ShardBaseNum)
this.User = section.String("user", "")
this.Pass = section.String("pass", "")
this.ReplicaSet = section.String("replicaSet", "")
if this.Host == "" ||
this.Port == "" ||
this.Kind == "" ||
this.DbName == "" {
panic("required field missing")
}
log.Debug("mongodb server: %+v", *this)
}
func (this *ArchiveInput) Init(config *conf.Conf) {
this.rootDir = config.String("root_dir", "")
this.ident = config.String("ident", "")
if this.ident == "" {
panic("empty ident")
}
this.project = config.String("project", "rs")
this.workerNChan = make(chan int, config.Int("concurrent_num", 20))
this.chkpnt = als.NewFileCheckpoint(config.String("chkpntfile", ""))
this.ignores = config.StringList("ignores", nil)
}
func (this *EsBufferFilter) Init(config *conf.Conf) {
this.ident = config.String("ident", "")
if this.ident == "" {
panic("empty ident")
}
this.stopChan = make(chan interface{})
this.wokers = make([]*esBufferWorker, 0, 10)
for i := 0; i < len(config.List("workers", nil)); i++ {
section, err := config.Section(fmt.Sprintf("workers[%d]", i))
if err != nil {
panic(err)
}
worker := new(esBufferWorker)
worker.init(section, this.ident, this.stopChan)
this.wokers = append(this.wokers, worker)
}
}
func (this *cardinalityConverter) load(section *conf.Conf) {
this.logPrefix = section.String("log_prefix", "")
this.project = section.String("project", "")
this.fields = make([]cardinalityField, 0, 5)
for i := 0; i < len(section.List("fields", nil)); i++ {
keyPrefix := fmt.Sprintf("fields[%d].", i)
field := cardinalityField{}
field.key = section.String(keyPrefix+"key", "")
field.typ = section.String(keyPrefix+"type", "string")
field.intervals = section.StringList(keyPrefix+"intervals", nil)
this.fields = append(this.fields, field)
}
}
func (this *EsFilter) Init(config *conf.Conf) {
this.ident = config.String("ident", "")
if this.ident == "" {
panic("empty ident")
}
this.converters = make([]esConverter, 0, 10)
this.indexPattern = config.String("index_pattern", "")
for i := 0; i < len(config.List("converts", nil)); i++ {
section, err := config.Section(fmt.Sprintf("%s[%d]", "converts", i))
if err != nil {
panic(err)
}
c := esConverter{}
c.load(section)
this.converters = append(this.converters, c)
}
geodbFile := config.String("geodbfile", "")
if err := als.LoadGeoDb(geodbFile); err != nil {
panic(err)
}
globals := engine.Globals()
if globals.Verbose {
globals.Printf("Loaded geodb %s\n", geodbFile)
}
}
func (this *table) init(config *conf.Conf) error {
this.properties = make([]property, 0, 10)
this.name = config.String("name", "")
if this.name == "" {
return errors.New("empty table name")
}
for i := 0; i < len(config.List("props", nil)); i++ {
section, err := config.Section(fmt.Sprintf("props[%d]", i))
if err != nil {
return err
}
p := property{}
p.name = section.String("name", "")
p.transient = section.Bool("transient", true)
p.dataType = section.String("type", "")
this.properties = append(this.properties, p)
}
return nil
}
func (this *ConfigMysql) LoadConfig(cf *conf.Conf) {
this.GlobalPools = make(map[string]bool)
for _, p := range cf.StringList("global_pools", nil) {
this.GlobalPools[p] = true
}
this.ShardStrategy = cf.String("shard_strategy", "standard")
this.MaxIdleTime = cf.Duration("max_idle_time", 0)
this.Timeout = cf.Duration("timeout", 10*time.Second)
this.AllowNullableColumns = cf.Bool("allow_nullable_columns", true)
this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 2)
this.MaxConnsPerServer = cf.Int("max_conns_per_server",
this.MaxIdleConnsPerServer*5)
this.CachePrepareStmtMaxItems = cf.Int("cache_prepare_stmt_max_items", 0)
this.HeartbeatInterval = cf.Int("heartbeat_interval", 120)
this.CacheStore = cf.String("cache_store", "mem")
this.CacheStoreMemMaxItems = cf.Int("cache_store_mem_max_items", 10<<20)
this.CacheStoreRedisPool = cf.String("cache_store_redis_pool", "db_cache")
this.CacheKeyHash = cf.Bool("cache_key_hash", false)
this.LookupPool = cf.String("lookup_pool", "ShardLookup")
this.JsonMergeMaxOutstandingItems = cf.Int("json_merge_max_outstanding_items", 8<<20)
this.LookupCacheMaxItems = cf.Int("lookup_cache_max_items", 1<<20)
section, err := cf.Section("breaker")
if err == nil {
this.Breaker.loadConfig(section)
}
section, err = cf.Section("lookup_tables")
if err == nil {
this.lookupTables = *section
} else {
panic(err)
}
this.Servers = make(map[string]*ConfigMysqlServer)
for i := 0; i < len(cf.List("servers", nil)); i++ {
section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
if err != nil {
panic(err)
}
server := new(ConfigMysqlServer)
server.conf = this
server.loadConfig(section)
this.Servers[server.Pool] = server
}
log.Debug("mysql conf: %+v", *this)
}
func (this *alarmProjectConf) fromConfig(config *conf.Conf) {
this.name = config.String("name", "")
if this.name == "" {
panic("project has no 'name'")
}
mailSection, err := config.Section("alarm_email")
if err == nil {
this.mailConf = alarmProjectMailConf{}
this.mailConf.severityPoolSize = mailSection.Int("severity_pool_size", 100)
this.mailConf.severityThreshold = mailSection.Int("severity_threshold", 8)
this.mailConf.suppressHours = mailSection.IntList("suppress_hours", nil)
this.mailConf.recipients = mailSection.String("recipients", "")
if this.mailConf.recipients == "" {
panic("mail alarm can't have no recipients")
}
this.mailConf.interval = mailSection.Int("interval", 300)
}
this.emailChan = make(chan alarmMailMessage)
workersMutex := new(sync.Mutex)
this.workers = make(map[string]*alarmWorker)
for i := 0; i < len(config.List("workers", nil)); i++ {
section, err := config.Section(fmt.Sprintf("workers[%d]", i))
if err != nil {
panic(err)
}
worker := &alarmWorker{projName: this.name,
emailChan: this.emailChan, workersMutex: workersMutex}
worker.init(section, this.stopChan)
this.workers[worker.conf.camelName] = worker
}
if len(this.workers) == 0 {
panic(fmt.Sprintf("%s empty 'workers'", this.name))
}
}
func (this *ConfigMemcacheServer) loadConfig(section *conf.Conf) {
this.Host = section.String("host", "")
if this.Host == "" {
panic("Empty memcache server host")
}
this.Port = section.String("port", "")
if this.Port == "" {
panic("Empty memcache server port")
}
this.Pool = section.String("pool", "default")
}
func (this *pluginCommons) load(section *conf.Conf) {
this.name = section.String("name", "")
if this.name == "" {
pretty.Printf("%# v\n", *section)
panic(fmt.Sprintf("invalid plugin config: %v", *section))
}
this.class = section.String("class", "")
if this.class == "" {
this.class = this.name
}
this.comment = section.String("comment", "")
this.ticker = section.Int("ticker_interval", Globals().TickerLength)
this.disabled = section.Bool("disabled", false)
}
func (this *alarmWorkerConfigField) init(config *conf.Conf) {
this.name = config.String("name", "")
if this.name == "" {
panic("alarm worker field name can't be empty")
}
this.typ = config.String("type", als.KEY_TYPE_STRING)
this.parser = config.String("parser", "")
this.ignores = config.StringList("ignores", nil)
this._regexIgnores = make([]*regexp.Regexp, 0)
// build the precompiled regex matcher
for _, ignore := range this.ignores {
if strings.HasPrefix(ignore, "regex:") {
pattern := strings.TrimSpace(ignore[6:])
r, err := regexp.Compile(pattern)
if err != nil {
panic(err)
}
this._regexIgnores = append(this._regexIgnores, r)
}
}
this.normalizers = config.StringList("normalizers", nil)
}