Ejemplo n.º 1
0
func (this *EsFilter) Init(config *conf.Conf) {
	this.ident = config.String("ident", "")
	if this.ident == "" {
		panic("empty ident")
	}
	this.converters = make([]esConverter, 0, 10)
	this.indexPattern = config.String("index_pattern", "")
	for i := 0; i < len(config.List("converts", nil)); i++ {
		section, err := config.Section(fmt.Sprintf("%s[%d]", "converts", i))
		if err != nil {
			panic(err)
		}

		c := esConverter{}
		c.load(section)
		this.converters = append(this.converters, c)
	}

	geodbFile := config.String("geodbfile", "")
	if err := als.LoadGeoDb(geodbFile); err != nil {
		panic(err)
	}
	globals := engine.Globals()
	if globals.Verbose {
		globals.Printf("Loaded geodb %s\n", geodbFile)
	}
}
Ejemplo n.º 2
0
func (this *ConfigRedis) LoadConfig(cf *conf.Conf) {
	section, err := cf.Section("breaker")
	if err == nil {
		this.Breaker.loadConfig(section)
	}

	this.Servers = make(map[string]map[string]*ConfigRedisServer)
	for i := 0; i < len(cf.List("pools", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("pools[%d]", i))
		if err != nil {
			panic(err)
		}

		pool := section.String("name", "")
		if pool == "" {
			panic("Empty redis pool name")
		}

		this.Servers[pool] = make(map[string]*ConfigRedisServer)

		// get servers in each pool
		for j := 0; j < len(section.List("servers", nil)); j++ {
			server, err := section.Section(fmt.Sprintf("servers[%d]", j))
			if err != nil {
				panic(err)
			}

			redisServer := new(ConfigRedisServer)
			redisServer.loadConfig(server)
			this.Servers[pool][redisServer.Addr] = redisServer
		}
	}

	log.Debug("redis conf: %+v", *this)
}
Ejemplo n.º 3
0
func (this *ConfigMongodb) LoadConfig(cf *conf.Conf) {
	this.ShardBaseNum = cf.Int("shard_base_num", 100000)
	this.DebugProtocol = cf.Bool("debug_protocol", false)
	this.DebugHeartbeat = cf.Bool("debug_heartbeat", false)
	this.ShardStrategy = cf.String("shard_strategy", "legacy")
	this.ConnectTimeout = cf.Duration("connect_timeout", 4*time.Second)
	this.IoTimeout = cf.Duration("io_timeout", 30*time.Second)
	this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 2)
	this.MaxConnsPerServer = cf.Int("max_conns_per_server",
		this.MaxIdleConnsPerServer*5)
	this.HeartbeatInterval = cf.Int("heartbeat_interval", 120)
	section, err := cf.Section("breaker")
	if err == nil {
		this.Breaker.loadConfig(section)
	}
	this.Servers = make(map[string]*ConfigMongodbServer)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMongodbServer)
		server.ShardBaseNum = this.ShardBaseNum
		server.loadConfig(section)
		this.Servers[server.Pool] = server
	}

	log.Debug("mongodb conf: %+v", *this)
}
Ejemplo n.º 4
0
func (this *ConfigMemcache) LoadConfig(cf *conf.Conf) {
	this.Servers = make(map[string]*ConfigMemcacheServer)
	this.HashStrategy = cf.String("hash_strategy", "standard")
	this.Timeout = cf.Duration("timeout", 4*time.Second)
	this.ReplicaN = cf.Int("replica_num", 1)
	section, err := cf.Section("breaker")
	if err == nil {
		this.Breaker.loadConfig(section)
	}
	this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 3)
	this.MaxConnsPerServer = cf.Int("max_conns_per_server",
		this.MaxIdleConnsPerServer*10)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMemcacheServer)
		server.loadConfig(section)
		this.Servers[server.Address()] = server
	}

	log.Debug("memcache conf: %+v", *this)
}
Ejemplo n.º 5
0
func (this *cardinalityConverter) load(section *conf.Conf) {
	this.logPrefix = section.String("log_prefix", "")
	this.project = section.String("project", "")
	this.fields = make([]cardinalityField, 0, 5)
	for i := 0; i < len(section.List("fields", nil)); i++ {
		keyPrefix := fmt.Sprintf("fields[%d].", i)
		field := cardinalityField{}
		field.key = section.String(keyPrefix+"key", "")
		field.typ = section.String(keyPrefix+"type", "string")
		field.intervals = section.StringList(keyPrefix+"intervals", nil)
		this.fields = append(this.fields, field)
	}
}
Ejemplo n.º 6
0
func (this *ConfigMemcache) loadConfig(cf *conf.Conf) {
	this.Servers = make(map[string]*ConfigMemcacheServer)
	this.HashStrategy = cf.String("hash_strategy", "standard")
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMemcacheServer)
		server.loadConfig(section)
		this.Servers[server.Address()] = server
	}

	log.Debug("memcache: %+v", *this)
}
Ejemplo n.º 7
0
func (this *CardinalityFilter) Init(config *conf.Conf) {
	this.ident = config.String("ident", "")
	if this.ident == "" {
		panic("empty ident")
	}
	for i := 0; i < len(config.List("converts", nil)); i++ {
		section, err := config.Section(fmt.Sprintf("%s[%d]", "converts", i))
		if err != nil {
			panic(err)
		}

		c := cardinalityConverter{}
		c.load(section)
		this.converters = append(this.converters, c)
	}
}
Ejemplo n.º 8
0
func (this *ConfigMongodb) loadConfig(cf *conf.Conf) {
	this.ShardBaseNum = cf.Int("shard_base_num", 100000)
	this.Servers = make(map[string]*ConfigMongodbServer)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMongodbServer)
		server.loadConfig(section)
		this.Servers[server.ShardName] = server
	}

	log.Debug("mongodb: %+v", *this)
}
Ejemplo n.º 9
0
func (this *ConfigMysql) LoadConfig(cf *conf.Conf) {
	this.GlobalPools = make(map[string]bool)
	for _, p := range cf.StringList("global_pools", nil) {
		this.GlobalPools[p] = true
	}
	this.ShardStrategy = cf.String("shard_strategy", "standard")
	this.MaxIdleTime = cf.Duration("max_idle_time", 0)
	this.Timeout = cf.Duration("timeout", 10*time.Second)
	this.AllowNullableColumns = cf.Bool("allow_nullable_columns", true)
	this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 2)
	this.MaxConnsPerServer = cf.Int("max_conns_per_server",
		this.MaxIdleConnsPerServer*5)
	this.CachePrepareStmtMaxItems = cf.Int("cache_prepare_stmt_max_items", 0)
	this.HeartbeatInterval = cf.Int("heartbeat_interval", 120)
	this.CacheStore = cf.String("cache_store", "mem")
	this.CacheStoreMemMaxItems = cf.Int("cache_store_mem_max_items", 10<<20)
	this.CacheStoreRedisPool = cf.String("cache_store_redis_pool", "db_cache")
	this.CacheKeyHash = cf.Bool("cache_key_hash", false)
	this.LookupPool = cf.String("lookup_pool", "ShardLookup")
	this.JsonMergeMaxOutstandingItems = cf.Int("json_merge_max_outstanding_items", 8<<20)
	this.LookupCacheMaxItems = cf.Int("lookup_cache_max_items", 1<<20)
	section, err := cf.Section("breaker")
	if err == nil {
		this.Breaker.loadConfig(section)
	}
	section, err = cf.Section("lookup_tables")
	if err == nil {
		this.lookupTables = *section
	} else {
		panic(err)
	}

	this.Servers = make(map[string]*ConfigMysqlServer)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMysqlServer)
		server.conf = this
		server.loadConfig(section)
		this.Servers[server.Pool] = server
	}

	log.Debug("mysql conf: %+v", *this)
}
Ejemplo n.º 10
0
func (this *alarmWorkerConfig) init(config *conf.Conf) {
	this.camelName = config.String("camel_name", "")
	if this.camelName == "" {
		panic("empty 'camel_name'")
	}

	this.title = config.String("title", "")
	if this.title == "" {
		this.title = this.camelName
	}
	this.colors = config.StringList("colors", nil)
	this.printFormat = config.String("printf", "")
	this.instantFormat = config.String("iprintf", "")
	if this.printFormat == "" && this.instantFormat == "" {
		panic(fmt.Sprintf("%s empty 'printf' and 'iprintf'", this.title))
	}
	this.severity = config.Int("severity", 1)
	this.windowSize = time.Duration(config.Int("window_size", 0)) * time.Second
	this.showSummary = config.Bool("show_summary", false)
	this.beepThreshold = config.Int("beep_threshold", 0)
	this.abnormalBase = config.Int("abnormal_base", 10)
	this.abnormalSeverityFactor = config.Int("abnormal_severity_factor", 2)
	this.abnormalPercent = config.Float("abnormal_percent", 1.5)
	this.dbName = config.String("dbname", "")
	this.tableName = this.dbName // table name is db name
	this.createTable = config.String("create_table", "")
	this.insertStmt = config.String("insert_stmt", "")
	this.statsStmt = config.String("stats_stmt", "")

	this.fields = make([]alarmWorkerConfigField, 0, 5)
	for i := 0; i < len(config.List("fields", nil)); i++ {
		section, err := config.Section(fmt.Sprintf("fields[%d]", i))
		if err != nil {
			panic(err)
		}

		field := alarmWorkerConfigField{}
		field.init(section)
		this.fields = append(this.fields, field)
	}
	if len(this.fields) == 0 {
		panic(fmt.Sprintf("%s empty 'fields'", this.title))
	}
}
Ejemplo n.º 11
0
func LoadPlugins(cf *conf.Conf) {
	for i := 0; i < PackRecyclePoolSize; i++ {
		pack := NewPipelinePack(packRecycleChan)
		packRecycleChan <- pack
	}
	for i := 0; i < len(cf.List("plugins", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("plugins[%d]", i))
		if err != nil {
			panic(err)
		}

		name := section.String("name", "")
		if name == "" {
			panic("empty plugin name")
		}

		loadOnePlugin(name, section)
	}
}
Ejemplo n.º 12
0
func (this *EsBufferFilter) Init(config *conf.Conf) {
	this.ident = config.String("ident", "")
	if this.ident == "" {
		panic("empty ident")
	}

	this.stopChan = make(chan interface{})
	this.wokers = make([]*esBufferWorker, 0, 10)
	for i := 0; i < len(config.List("workers", nil)); i++ {
		section, err := config.Section(fmt.Sprintf("workers[%d]", i))
		if err != nil {
			panic(err)
		}

		worker := new(esBufferWorker)
		worker.init(section, this.ident, this.stopChan)
		this.wokers = append(this.wokers, worker)
	}
}
Ejemplo n.º 13
0
func (this *ConfigMongodb) loadConfig(cf *conf.Conf) {
	this.ShardBaseNum = cf.Int("shard_base_num", 100000)
	this.ConnectTimeout = cf.Int("connect_timeout", 4)
	this.IoTimeout = cf.Int("io_timeout", 30)
	this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 2)
	this.HeartbeatInterval = cf.Int("heartbeat_interval", 120)
	this.Servers = make(map[string]*ConfigMongodbServer)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMongodbServer)
		server.ShardBaseNum = this.ShardBaseNum
		server.loadConfig(section)
		this.Servers[server.Kind] = server
	}

	log.Debug("mongodb: %+v", *this)
}
Ejemplo n.º 14
0
func (this *AlarmOutput) Init(config *conf.Conf) {
	this.stopChan = make(chan interface{})
	this.projects = make(map[string]alarmProjectConf)
	for i := 0; i < len(config.List("projects", nil)); i++ {
		section, err := config.Section(fmt.Sprintf("projects[%d]", i))
		if err != nil {
			panic(err)
		}

		project := alarmProjectConf{}
		project.stopChan = this.stopChan
		project.fromConfig(section)
		if _, present := this.projects[project.name]; present {
			panic("dup project: " + project.name)
		}
		this.projects[project.name] = project
	}
	if len(this.projects) == 0 {
		panic("empty projects")
	}
}
Ejemplo n.º 15
0
func (this *table) init(config *conf.Conf) error {
	this.properties = make([]property, 0, 10)
	this.name = config.String("name", "")
	if this.name == "" {
		return errors.New("empty table name")
	}

	for i := 0; i < len(config.List("props", nil)); i++ {
		section, err := config.Section(fmt.Sprintf("props[%d]", i))
		if err != nil {
			return err
		}

		p := property{}
		p.name = section.String("name", "")
		p.transient = section.Bool("transient", true)
		p.dataType = section.String("type", "")
		this.properties = append(this.properties, p)
	}

	return nil
}
Ejemplo n.º 16
0
func (this *alarmProjectConf) fromConfig(config *conf.Conf) {
	this.name = config.String("name", "")
	if this.name == "" {
		panic("project has no 'name'")
	}

	mailSection, err := config.Section("alarm_email")
	if err == nil {
		this.mailConf = alarmProjectMailConf{}
		this.mailConf.severityPoolSize = mailSection.Int("severity_pool_size", 100)
		this.mailConf.severityThreshold = mailSection.Int("severity_threshold", 8)
		this.mailConf.suppressHours = mailSection.IntList("suppress_hours", nil)
		this.mailConf.recipients = mailSection.String("recipients", "")
		if this.mailConf.recipients == "" {
			panic("mail alarm can't have no recipients")
		}
		this.mailConf.interval = mailSection.Int("interval", 300)
	}

	this.emailChan = make(chan alarmMailMessage)
	workersMutex := new(sync.Mutex)
	this.workers = make(map[string]*alarmWorker)
	for i := 0; i < len(config.List("workers", nil)); i++ {
		section, err := config.Section(fmt.Sprintf("workers[%d]", i))
		if err != nil {
			panic(err)
		}

		worker := &alarmWorker{projName: this.name,
			emailChan: this.emailChan, workersMutex: workersMutex}
		worker.init(section, this.stopChan)
		this.workers[worker.conf.camelName] = worker
	}
	if len(this.workers) == 0 {
		panic(fmt.Sprintf("%s empty 'workers'", this.name))
	}
}
Ejemplo n.º 17
0
// recursively
func handleSection(section *conf.Conf) {
	ident := section.String(IDENT, "")
	if ident != "" {
		if _, present := graph[ident]; present {
			fmt.Printf("ident[%s]duplicated\n", ident)
			os.Exit(1)
		}

		ie := &identEntry{}
		ie.matches = make([]string, 0, 10)
		ie.disabled = section.Bool(DISABLED, false)
		if section.StringList(MATCH, nil) == nil {
			ie.isInput = true
		}

		graph[ident] = ie
		pluginName := section.String(NAME, "")
		if pluginName != "" {
			identName[pluginName] = ident
		}
	}

	matches := section.StringList(MATCH, nil)
	if matches != nil {
		pluginName := section.String(NAME, "")
		if pluginName == "" {
			fmt.Printf("plugin match %v has no 'name' key\n", matches)
			os.Exit(1)
		}

		for _, id := range matches {
			if _, present := graph[id]; present {
				graph[id].matches = append(graph[id].matches, pluginName)
			} else {
				fmt.Printf("%15s -> %s\n", id, pluginName)
			}
		}
	}

	sub := section.Interface("", nil).(map[string]interface{})
	if sub == nil {
		return
	}

	for k, v := range sub {
		if x, ok := v.([]interface{}); ok {
			switch x[0].(type) {
			case string, float64, int, bool, time.Time:
				// this section will never find 'ident'
				continue
			}

			for i := 0; i < len(section.List(k, nil)); i++ {
				key := fmt.Sprintf("%s[%d]", k, i)
				sec, err := section.Section(key)
				if err != nil {
					continue
				}

				handleSection(sec)
			}
		}
	}
}