Example #1
0
func (this *ConfigMongodbServer) loadConfig(section *conf.Conf) {
	this.Pool = section.String("pool", "")
	this.Host = section.String("host", "")
	this.Port = section.String("port", "27017")
	this.DbName = section.String("db", "")
	this.ShardBaseNum = section.Int("shard_base_num", this.ShardBaseNum)
	this.User = section.String("user", "")
	this.Pass = section.String("pass", "")
	this.ReplicaSet = section.String("replicaSet", "")
	if this.Host == "" ||
		this.Port == "" ||
		this.Pool == "" ||
		this.DbName == "" {
		panic("required field missing")
	}

	// http://docs.mongodb.org/manual/reference/connection-string/
	this.uri = "mongodb://" + this.Host + ":" + this.Port + "/"
	if this.DbName != "" {
		this.uri += this.DbName + "/"
	}
	if this.ReplicaSet != "" {
		this.uri += "?replicaSet=" + this.ReplicaSet
	}

}
Example #2
0
func (this *ConfigProxy) LoadConfig(selfAddr string, cf *conf.Conf) {
	if selfAddr == "" {
		panic("proxy self addr unknown")
	}
	this.PoolCapacity = cf.Int("pool_capacity", 10)
	this.IdleTimeout = cf.Duration("idle_timeout", 0)
	this.IoTimeout = cf.Duration("io_timeout", time.Second*10)
	this.BorrowTimeout = cf.Duration("borrow_timeout", time.Second*10)
	this.DiagnosticInterval = cf.Duration("diagnostic_interval", time.Second*5)
	this.TcpNoDelay = cf.Bool("tcp_nodelay", true)
	this.BufferSize = cf.Int("buffer_size", 4<<10)
	this.SelfAddr = selfAddr
	parts := strings.SplitN(this.SelfAddr, ":", 2)
	if parts[0] == "" {
		// auto get local ip when self_addr like ":9001"
		ips, _ := ip.LocalIpv4Addrs()
		if len(ips) == 0 {
			panic("cannot get local ip address")
		}

		this.SelfAddr = ips[0] + ":" + parts[1]
	}

	log.Debug("proxy conf: %+v", *this)
}
Example #3
0
func (this *SkyOutput) Init(config *conf.Conf) {
	const TYPE_SEP = ":"
	this.uidFieldType, this.actionFieldType = als.KEY_TYPE_INT, als.KEY_TYPE_STRING
	this.uidField = config.String("uid_field", "_log_info.uid")
	if strings.Contains(this.uidField, TYPE_SEP) {
		p := strings.SplitN(this.uidField, TYPE_SEP, 2)
		this.uidField, this.uidFieldType = p[0], p[1]
	}
	this.actionField = config.String("action_field", "action")
	if this.actionField == "" {
		panic("empty action field")
	}
	if strings.Contains(this.actionField, TYPE_SEP) {
		p := strings.SplitN(this.actionField, TYPE_SEP, 2)
		this.actionField, this.actionFieldType = p[0], p[1]
	}

	this.project = config.String("project", "")
	var (
		host string = config.String("host", "localhost")
		port int    = config.Int("port", 8585)
	)
	client := sky.NewClient(host)
	client.Port = port

	if !client.Ping() {
		panic(fmt.Sprintf("sky server not running: %s:%d", host, port))
	}

	this.table, _ = client.GetTable(config.String("table", ""))
	if this.table == nil {
		panic("must create table in advance")
	}

}
Example #4
0
func (this *esBufferWorker) init(config *conf.Conf, ident string,
	stopChan chan interface{}) {
	this.camelName = config.String("camel_name", "")
	if this.camelName == "" {
		panic("empty camel_name")
	}

	this.ident = ident
	this.stopChan = stopChan
	this.interval = time.Duration(config.Int("interval", 10)) * time.Second
	this.projectName = config.String("project", "")
	this.indexPattern = config.String("index_pattern", "@ym")
	this.expression = config.String("expression", "count")
	if this.expression != "count" {
		this.fieldName = config.String("field_name", "")
		if this.fieldName == "" {
			panic("empty field_name")
		}
		this.fieldType = config.String("field_type", "float")
	}

	this.summary = stats.Summary{}

	// prefill the es field name
	switch this.expression {
	case "count":
		this.esField = "count"
	default:
		this.esField = this.expression + "_" + this.fieldName
	}
	this.esType = this.camelName + "_" + this.expression
}
Example #5
0
func (this *ConfigLock) LoadConfig(cf *conf.Conf) {
	this.MaxItems = cf.Int("max_items", 1<<20)
	this.Expires = cf.Duration("expires", time.Second*10)

	this.enabled = true

	log.Debug("lock conf: %+v", *this)
}
Example #6
0
func (this *ConfigRedisServer) loadConfig(cf *conf.Conf) {
	this.Addr = cf.String("addr", "")
	if this.Addr == "" {
		panic("Empty redis server addr")
	}
	this.MaxIdle = cf.Int("max_idle", 10)
	this.MaxActive = cf.Int("max_active", this.MaxIdle*2)
	this.IdleTimeout = cf.Duration("idle_timeout", 10*time.Minute)
}
Example #7
0
func (this *ArchiveInput) Init(config *conf.Conf) {
	this.rootDir = config.String("root_dir", "")
	this.ident = config.String("ident", "")
	if this.ident == "" {
		panic("empty ident")
	}
	this.project = config.String("project", "rs")
	this.workerNChan = make(chan int, config.Int("concurrent_num", 20))
	this.chkpnt = als.NewFileCheckpoint(config.String("chkpntfile", ""))
	this.ignores = config.StringList("ignores", nil)
}
Example #8
0
func (this *configRpc) loadConfig(section *conf.Conf) {
	this.listenAddr = section.String("listen_addr", "")
	if this.listenAddr == "" {
		panic("Empty listen_addr")
	}

	this.clientTimeout = time.Duration(section.Int("client_timeout", 0)) * time.Second
	this.framed = section.Bool("framed", false)
	this.protocol = section.String("protocol", "binary")

	log.Debug("rpc: %+v", *this)
}
Example #9
0
func (this *EsOutput) Init(config *conf.Conf) {
	this.stopChan = make(chan bool)
	api.Domain = config.String("domain", "localhost")
	this.counters = sortedmap.NewSortedMap()
	api.Port = config.String("port", "9200")
	this.reportInterval = time.Duration(config.Int("report_interval", 30)) * time.Second
	this.showProgress = config.Bool("show_progress", true)
	this.flushInterval = time.Duration(config.Int("flush_interval", 30)) * time.Second
	this.bulkMaxConn = config.Int("bulk_max_conn", 20)
	this.bulkMaxDocs = config.Int("bulk_max_docs", 100)
	this.bulkMaxBuffer = config.Int("bulk_max_buffer", 10<<20) // 10 MB
	this.dryRun = config.Bool("dryrun", false)
}
Example #10
0
func (this *ConfigMongodb) LoadConfig(cf *conf.Conf) {
	this.ShardBaseNum = cf.Int("shard_base_num", 100000)
	this.DebugProtocol = cf.Bool("debug_protocol", false)
	this.DebugHeartbeat = cf.Bool("debug_heartbeat", false)
	this.ShardStrategy = cf.String("shard_strategy", "legacy")
	this.ConnectTimeout = cf.Duration("connect_timeout", 4*time.Second)
	this.IoTimeout = cf.Duration("io_timeout", 30*time.Second)
	this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 2)
	this.MaxConnsPerServer = cf.Int("max_conns_per_server",
		this.MaxIdleConnsPerServer*5)
	this.HeartbeatInterval = cf.Int("heartbeat_interval", 120)
	section, err := cf.Section("breaker")
	if err == nil {
		this.Breaker.loadConfig(section)
	}
	this.Servers = make(map[string]*ConfigMongodbServer)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMongodbServer)
		server.ShardBaseNum = this.ShardBaseNum
		server.loadConfig(section)
		this.Servers[server.Pool] = server
	}

	log.Debug("mongodb conf: %+v", *this)
}
Example #11
0
func (this *pluginCommons) load(section *conf.Conf) {
	this.name = section.String("name", "")
	if this.name == "" {
		pretty.Printf("%# v\n", *section)
		panic(fmt.Sprintf("invalid plugin config: %v", *section))
	}

	this.class = section.String("class", "")
	if this.class == "" {
		this.class = this.name
	}
	this.comment = section.String("comment", "")
	this.ticker = section.Int("ticker_interval", Globals().TickerLength)
	this.disabled = section.Bool("disabled", false)
}
Example #12
0
func (this *ConfigMongodb) loadConfig(cf *conf.Conf) {
	this.ShardBaseNum = cf.Int("shard_base_num", 100000)
	this.Servers = make(map[string]*ConfigMongodbServer)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMongodbServer)
		server.loadConfig(section)
		this.Servers[server.ShardName] = server
	}

	log.Debug("mongodb: %+v", *this)
}
Example #13
0
func (this *ConfigMongodbServer) loadConfig(section *conf.Conf) {
	this.Kind = section.String("kind", "")
	this.Host = section.String("host", "")
	this.Port = section.String("port", "27017")
	this.DbName = section.String("db", "")
	this.ShardBaseNum = section.Int("shard_base_num", this.ShardBaseNum)
	this.User = section.String("user", "")
	this.Pass = section.String("pass", "")
	this.ReplicaSet = section.String("replicaSet", "")
	if this.Host == "" ||
		this.Port == "" ||
		this.Kind == "" ||
		this.DbName == "" {
		panic("required field missing")
	}

	log.Debug("mongodb server: %+v", *this)
}
Example #14
0
func (this *ConfigMemcache) loadConfig(cf *conf.Conf) {
	this.Servers = make(map[string]*ConfigMemcacheServer)
	this.HashStrategy = cf.String("hash_strategy", "standard")
	this.Timeout = cf.Int("timeout", 4)
	this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 3)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMemcacheServer)
		server.loadConfig(section)
		this.Servers[server.Address()] = server
	}

	log.Debug("memcache: %+v", *this)
}
Example #15
0
func (this *alarmWorkerConfig) init(config *conf.Conf) {
	this.camelName = config.String("camel_name", "")
	if this.camelName == "" {
		panic("empty 'camel_name'")
	}

	this.title = config.String("title", "")
	if this.title == "" {
		this.title = this.camelName
	}
	this.colors = config.StringList("colors", nil)
	this.printFormat = config.String("printf", "")
	this.instantFormat = config.String("iprintf", "")
	if this.printFormat == "" && this.instantFormat == "" {
		panic(fmt.Sprintf("%s empty 'printf' and 'iprintf'", this.title))
	}
	this.severity = config.Int("severity", 1)
	this.windowSize = time.Duration(config.Int("window_size", 0)) * time.Second
	this.showSummary = config.Bool("show_summary", false)
	this.beepThreshold = config.Int("beep_threshold", 0)
	this.abnormalBase = config.Int("abnormal_base", 10)
	this.abnormalSeverityFactor = config.Int("abnormal_severity_factor", 2)
	this.abnormalPercent = config.Float("abnormal_percent", 1.5)
	this.dbName = config.String("dbname", "")
	this.tableName = this.dbName // table name is db name
	this.createTable = config.String("create_table", "")
	this.insertStmt = config.String("insert_stmt", "")
	this.statsStmt = config.String("stats_stmt", "")

	this.fields = make([]alarmWorkerConfigField, 0, 5)
	for i := 0; i < len(config.List("fields", nil)); i++ {
		section, err := config.Section(fmt.Sprintf("fields[%d]", i))
		if err != nil {
			panic(err)
		}

		field := alarmWorkerConfigField{}
		field.init(section)
		this.fields = append(this.fields, field)
	}
	if len(this.fields) == 0 {
		panic(fmt.Sprintf("%s empty 'fields'", this.title))
	}
}
Example #16
0
func (this *ConfigMongodb) loadConfig(cf *conf.Conf) {
	this.ShardBaseNum = cf.Int("shard_base_num", 100000)
	this.ConnectTimeout = cf.Int("connect_timeout", 4)
	this.IoTimeout = cf.Int("io_timeout", 30)
	this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 2)
	this.HeartbeatInterval = cf.Int("heartbeat_interval", 120)
	this.Servers = make(map[string]*ConfigMongodbServer)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMongodbServer)
		server.ShardBaseNum = this.ShardBaseNum
		server.loadConfig(section)
		this.Servers[server.Kind] = server
	}

	log.Debug("mongodb: %+v", *this)
}
Example #17
0
func LoadServants(cf *conf.Conf) {
	Servants.WatchdogInterval = cf.Int("watchdog_interval", 60*10)
	Servants.PeersCooperate = cf.Bool("peers_cooperate", false)
	Servants.ProfilerMaxAnswerSize = cf.Int("prof_max_answer_size", 4<<10)
	Servants.ProfilerRate = cf.Int("profiler_rate", 1) // default 1/1000

	// mongodb section
	Servants.Mongodb = new(ConfigMongodb)
	section, err := cf.Section("mongodb")
	if err != nil {
		panic(err)
	}
	Servants.Mongodb.loadConfig(section)

	// memcached section
	Servants.Memcache = new(ConfigMemcache)
	section, err = cf.Section("memcache")
	if err != nil {
		panic(err)
	}
	Servants.Memcache.loadConfig(section)

	// lcache section
	Servants.Lcache = new(ConfigLcache)
	section, err = cf.Section("lcache")
	if err != nil {
		panic(err)
	}
	Servants.Lcache.loadConfig(section)
}
Example #18
0
func (this *ConfigRpc) LoadConfig(section *conf.Conf) {
	this.ListenAddr = section.String("listen_addr", "")
	if this.ListenAddr == "" {
		panic("Empty listen_addr")
	}

	this.SessionTimeout = section.Duration("session_timeout", 30*time.Second)
	this.IoTimeout = section.Duration("io_timeout", 2*time.Second)
	this.StatsOutputInterval = section.Duration("stats_output_interval", 10*time.Second)
	this.Framed = section.Bool("framed", false)
	this.BufferSize = section.Int("buffer_size", 4<<10)
	this.Protocol = section.String("protocol", "binary")
	this.PreforkMode = section.Bool("prefork_mode", false)
	this.MaxOutstandingSessions = section.Int("max_outstanding_sessions", 20000)
	this.HostMaxCallPerMinute = section.Int("host_max_call_per_minute", 100*60)

	log.Debug("rpc conf: %+v", *this)
}
Example #19
0
func (this *NetSenderOutput) Init(config *conf.Conf) {
	this.remoteAddr = config.String("remote_addr", ":9000")
	this.maxLineSize = config.Int("max_line_size", 8<<10)
}
Example #20
0
func (this *ConfigBreaker) loadConfig(cf *conf.Conf) {
	this.FailureAllowance = uint(cf.Int("failure_allowance", 5))
	this.RetryTimeout = cf.Duration("retry_interval", 10*time.Second)
}
Example #21
0
func (this *ConfigMysql) LoadConfig(cf *conf.Conf) {
	this.GlobalPools = make(map[string]bool)
	for _, p := range cf.StringList("global_pools", nil) {
		this.GlobalPools[p] = true
	}
	this.ShardStrategy = cf.String("shard_strategy", "standard")
	this.MaxIdleTime = cf.Duration("max_idle_time", 0)
	this.Timeout = cf.Duration("timeout", 10*time.Second)
	this.AllowNullableColumns = cf.Bool("allow_nullable_columns", true)
	this.MaxIdleConnsPerServer = cf.Int("max_idle_conns_per_server", 2)
	this.MaxConnsPerServer = cf.Int("max_conns_per_server",
		this.MaxIdleConnsPerServer*5)
	this.CachePrepareStmtMaxItems = cf.Int("cache_prepare_stmt_max_items", 0)
	this.HeartbeatInterval = cf.Int("heartbeat_interval", 120)
	this.CacheStore = cf.String("cache_store", "mem")
	this.CacheStoreMemMaxItems = cf.Int("cache_store_mem_max_items", 10<<20)
	this.CacheStoreRedisPool = cf.String("cache_store_redis_pool", "db_cache")
	this.CacheKeyHash = cf.Bool("cache_key_hash", false)
	this.LookupPool = cf.String("lookup_pool", "ShardLookup")
	this.JsonMergeMaxOutstandingItems = cf.Int("json_merge_max_outstanding_items", 8<<20)
	this.LookupCacheMaxItems = cf.Int("lookup_cache_max_items", 1<<20)
	section, err := cf.Section("breaker")
	if err == nil {
		this.Breaker.loadConfig(section)
	}
	section, err = cf.Section("lookup_tables")
	if err == nil {
		this.lookupTables = *section
	} else {
		panic(err)
	}

	this.Servers = make(map[string]*ConfigMysqlServer)
	for i := 0; i < len(cf.List("servers", nil)); i++ {
		section, err := cf.Section(fmt.Sprintf("servers[%d]", i))
		if err != nil {
			panic(err)
		}

		server := new(ConfigMysqlServer)
		server.conf = this
		server.loadConfig(section)
		this.Servers[server.Pool] = server
	}

	log.Debug("mysql conf: %+v", *this)
}
Example #22
0
func (this *ConfigLcache) loadConfig(cf *conf.Conf) {
	this.LruMaxItems = cf.Int("lru_max_items", 1<<30)

	log.Debug("lcache: %+v", *this)
}
Example #23
0
func (this *ConfigServant) LoadConfig(selfAddr string, cf *conf.Conf) {
	this.IdgenWorkerId = cf.Int("idgen_worker_id", 1)
	this.SessionMaxItems = cf.Int("session_max_items", 20<<10)
	this.CallSlowThreshold = cf.Duration("call_slow_threshold", 2*time.Second)
	this.StatsOutputInterval = cf.Duration("stats_output_interval", 10*time.Minute)
	this.ProfilerMaxBodySize = cf.Int("profiler_max_body_size", 1<<10)
	this.ProfilerRate = cf.Int("profiler_rate", 1) // default 1/1000

	// mongodb section
	this.Mongodb = new(ConfigMongodb)
	section, err := cf.Section("mongodb")
	if err == nil {
		this.Mongodb.LoadConfig(section)
	}

	this.Mysql = new(ConfigMysql)
	section, err = cf.Section("mysql")
	if err == nil {
		this.Mysql.LoadConfig(section)
	}

	this.Redis = new(ConfigRedis)
	section, err = cf.Section("redis")
	if err == nil {
		this.Redis.LoadConfig(section)
	}

	// memcached section
	this.Memcache = new(ConfigMemcache)
	section, err = cf.Section("memcache")
	if err == nil {
		this.Memcache.LoadConfig(section)
	}

	// lcache section
	this.Lcache = new(ConfigLcache)
	section, err = cf.Section("lcache")
	if err == nil {
		this.Lcache.LoadConfig(section)
	}

	// couchbase section
	this.Couchbase = new(ConfigCouchbase)
	section, err = cf.Section("couchbase")
	if err == nil {
		this.Couchbase.LoadConfig(section)
	}

	// proxy section
	this.Proxy = new(ConfigProxy)
	section, err = cf.Section("proxy")
	if err == nil {
		this.Proxy.LoadConfig(selfAddr, section)
	}

	this.Lock = new(ConfigLock)
	section, err = cf.Section("lock")
	if err == nil {
		this.Lock.LoadConfig(section)
	}

	log.Debug("servants conf: %+v", *this)
}
Example #24
0
func (this *RateLimiter) Init(cf *conf.Conf) {
	this.limit = cf.Int("limit", 2000)

	log.Debug("%T init: %+v", *this, *this)
}
Example #25
0
func (this *NetReceiverInput) Init(config *conf.Conf) {
	this.listenAddr = config.String("listen_addr", ":9000")
	this.maxLineSize = config.Int("max_line_size", 8<<10)
}
Example #26
0
func (this *ConfigLcache) LoadConfig(cf *conf.Conf) {
	this.MaxItems = cf.Int("max_items", 1<<30)

	log.Debug("lcache conf: %+v", *this)
}