func resolveConfig() (string, string, time.Duration, string, time.Duration, int, time.Duration, int, int) {
rawConfig, err := kafkaClient.LoadConfiguration("producers.properties")
if err != nil {
panic(err)
}
//zkConnect := rawConfig["zookeeper_connect"]
brokerConnect := rawConfig["broker_connect"]
topic := rawConfig["topic"]
//numPartitions, _ := strconv.Atoi(rawConfig["num_partitions"])
sleepTime, _ := time.ParseDuration(rawConfig["sleep_time"])
flushInterval, _ := time.ParseDuration(rawConfig["flush_interval"])
flushMsgCount, _ := strconv.Atoi(rawConfig["flush_msg_count"])
flushFrequency, _ := time.ParseDuration(rawConfig["flush_frequency"])
producerCount, _ := strconv.Atoi(rawConfig["producer_count"])
maxMessagesPerReq, _ := strconv.Atoi(rawConfig["max_messages_per_req"])
return brokerConnect, topic, sleepTime, rawConfig["graphite_connect"], flushInterval, flushMsgCount, flushFrequency, producerCount, maxMessagesPerReq
}
func resolveConfig() (*kafkaClient.ConsumerConfig, string, int, string, time.Duration) {
rawConfig, err := kafkaClient.LoadConfiguration("consumers.properties")
if err != nil {
panic("Failed to load configuration file")
}
logLevel := rawConfig["log_level"]
setLogLevel(logLevel)
numConsumers, _ := strconv.Atoi(rawConfig["num_consumers"])
zkTimeout, _ := time.ParseDuration(rawConfig["zookeeper_timeout"])
numWorkers, _ := strconv.Atoi(rawConfig["num_workers"])
maxWorkerRetries, _ := strconv.Atoi(rawConfig["max_worker_retries"])
workerBackoff, _ := time.ParseDuration(rawConfig["worker_backoff"])
workerRetryThreshold, _ := strconv.Atoi(rawConfig["worker_retry_threshold"])
workerConsideredFailedTimeWindow, _ := time.ParseDuration(rawConfig["worker_considered_failed_time_window"])
workerTaskTimeout, _ := time.ParseDuration(rawConfig["worker_task_timeout"])
workerManagersStopTimeout, _ := time.ParseDuration(rawConfig["worker_managers_stop_timeout"])
rebalanceBarrierTimeout, _ := time.ParseDuration(rawConfig["rebalance_barrier_timeout"])
rebalanceMaxRetries, _ := strconv.Atoi(rawConfig["rebalance_max_retries"])
rebalanceBackoff, _ := time.ParseDuration(rawConfig["rebalance_backoff"])
partitionAssignmentStrategy, _ := rawConfig["partition_assignment_strategy"]
excludeInternalTopics, _ := strconv.ParseBool(rawConfig["exclude_internal_topics"])
numConsumerFetchers, _ := strconv.Atoi(rawConfig["num_consumer_fetchers"])
fetchBatchSize, _ := strconv.Atoi(rawConfig["fetch_batch_size"])
fetchMessageMaxBytes, _ := strconv.Atoi(rawConfig["fetch_message_max_bytes"])
fetchMinBytes, _ := strconv.Atoi(rawConfig["fetch_min_bytes"])
fetchBatchTimeout, _ := time.ParseDuration(rawConfig["fetch_batch_timeout"])
requeueAskNextBackoff, _ := time.ParseDuration(rawConfig["requeue_ask_next_backoff"])
fetchWaitMaxMs, _ := strconv.Atoi(rawConfig["fetch_wait_max_ms"])
socketTimeout, _ := time.ParseDuration(rawConfig["socket_timeout"])
queuedMaxMessages, _ := strconv.Atoi(rawConfig["queued_max_messages"])
refreshLeaderBackoff, _ := time.ParseDuration(rawConfig["refresh_leader_backoff"])
fetchMetadataRetries, _ := strconv.Atoi(rawConfig["fetch_metadata_retries"])
fetchMetadataBackoff, _ := time.ParseDuration(rawConfig["fetch_metadata_backoff"])
time.ParseDuration(rawConfig["fetch_metadata_backoff"])
offsetsCommitMaxRetries, _ := strconv.Atoi(rawConfig["offsets_commit_max_retries"])
flushInterval, _ := time.ParseDuration(rawConfig["flush_interval"])
deploymentTimeout, _ := time.ParseDuration(rawConfig["deployment_timeout"])
zkConfig := kafkaClient.NewZookeeperConfig()
zkConfig.ZookeeperConnect = []string{rawConfig["zookeeper_connect"]}
zkConfig.ZookeeperTimeout = zkTimeout
config := kafkaClient.DefaultConsumerConfig()
config.Groupid = rawConfig["group_id"]
config.NumWorkers = numWorkers
config.MaxWorkerRetries = maxWorkerRetries
config.WorkerBackoff = workerBackoff
config.WorkerRetryThreshold = int32(workerRetryThreshold)
config.WorkerThresholdTimeWindow = workerConsideredFailedTimeWindow
config.WorkerTaskTimeout = workerTaskTimeout
config.WorkerManagersStopTimeout = workerManagersStopTimeout
config.BarrierTimeout = rebalanceBarrierTimeout
config.RebalanceMaxRetries = int32(rebalanceMaxRetries)
config.RebalanceBackoff = rebalanceBackoff
config.PartitionAssignmentStrategy = partitionAssignmentStrategy
config.ExcludeInternalTopics = excludeInternalTopics
config.NumConsumerFetchers = numConsumerFetchers
config.FetchBatchSize = fetchBatchSize
config.FetchMessageMaxBytes = int32(fetchMessageMaxBytes)
config.FetchMinBytes = int32(fetchMinBytes)
config.FetchBatchTimeout = fetchBatchTimeout
config.FetchTopicMetadataRetries = fetchMetadataRetries
config.FetchTopicMetadataBackoff = fetchMetadataBackoff
config.RequeueAskNextBackoff = requeueAskNextBackoff
config.FetchWaitMaxMs = int32(fetchWaitMaxMs)
config.SocketTimeout = socketTimeout
config.QueuedMaxMessages = int32(queuedMaxMessages)
config.RefreshLeaderBackoff = refreshLeaderBackoff
config.Coordinator = kafkaClient.NewZookeeperCoordinator(zkConfig)
config.AutoOffsetReset = rawConfig["auto_offset_reset"]
config.OffsetsCommitMaxRetries = offsetsCommitMaxRetries
config.DeploymentTimeout = deploymentTimeout
config.OffsetCommitInterval = 10 * time.Second
return config, rawConfig["topic"], numConsumers, rawConfig["graphite_connect"], flushInterval
}