func resolveConfig() (string, string, time.Duration, string, time.Duration, int, time.Duration, int, int) { rawConfig, err := kafkaClient.LoadConfiguration("producers.properties") if err != nil { panic(err) } //zkConnect := rawConfig["zookeeper_connect"] brokerConnect := rawConfig["broker_connect"] topic := rawConfig["topic"] //numPartitions, _ := strconv.Atoi(rawConfig["num_partitions"]) sleepTime, _ := time.ParseDuration(rawConfig["sleep_time"]) flushInterval, _ := time.ParseDuration(rawConfig["flush_interval"]) flushMsgCount, _ := strconv.Atoi(rawConfig["flush_msg_count"]) flushFrequency, _ := time.ParseDuration(rawConfig["flush_frequency"]) producerCount, _ := strconv.Atoi(rawConfig["producer_count"]) maxMessagesPerReq, _ := strconv.Atoi(rawConfig["max_messages_per_req"]) return brokerConnect, topic, sleepTime, rawConfig["graphite_connect"], flushInterval, flushMsgCount, flushFrequency, producerCount, maxMessagesPerReq }
func resolveConfig() (*kafkaClient.ConsumerConfig, string, int, string, time.Duration) { rawConfig, err := kafkaClient.LoadConfiguration("consumers.properties") if err != nil { panic("Failed to load configuration file") } numConsumers, _ := strconv.Atoi(rawConfig["num_consumers"]) zkTimeout, _ := time.ParseDuration(rawConfig["zookeeper_timeout"]) numWorkers, _ := strconv.Atoi(rawConfig["num_workers"]) maxWorkerRetries, _ := strconv.Atoi(rawConfig["max_worker_retries"]) workerBackoff, _ := time.ParseDuration(rawConfig["worker_backoff"]) workerRetryThreshold, _ := strconv.Atoi(rawConfig["worker_retry_threshold"]) workerConsideredFailedTimeWindow, _ := time.ParseDuration(rawConfig["worker_considered_failed_time_window"]) workerTaskTimeout, _ := time.ParseDuration(rawConfig["worker_task_timeout"]) workerManagersStopTimeout, _ := time.ParseDuration(rawConfig["worker_managers_stop_timeout"]) rebalanceBarrierTimeout, _ := time.ParseDuration(rawConfig["rebalance_barrier_timeout"]) rebalanceMaxRetries, _ := strconv.Atoi(rawConfig["rebalance_max_retries"]) rebalanceBackoff, _ := time.ParseDuration(rawConfig["rebalance_backoff"]) partitionAssignmentStrategy, _ := rawConfig["partition_assignment_strategy"] excludeInternalTopics, _ := strconv.ParseBool(rawConfig["exclude_internal_topics"]) numConsumerFetchers, _ := strconv.Atoi(rawConfig["num_consumer_fetchers"]) fetchBatchSize, _ := strconv.Atoi(rawConfig["fetch_batch_size"]) fetchMessageMaxBytes, _ := strconv.Atoi(rawConfig["fetch_message_max_bytes"]) fetchMinBytes, _ := strconv.Atoi(rawConfig["fetch_min_bytes"]) fetchBatchTimeout, _ := time.ParseDuration(rawConfig["fetch_batch_timeout"]) requeueAskNextBackoff, _ := time.ParseDuration(rawConfig["requeue_ask_next_backoff"]) fetchWaitMaxMs, _ := strconv.Atoi(rawConfig["fetch_wait_max_ms"]) socketTimeout, _ := time.ParseDuration(rawConfig["socket_timeout"]) queuedMaxMessages, _ := strconv.Atoi(rawConfig["queued_max_messages"]) refreshLeaderBackoff, _ := time.ParseDuration(rawConfig["refresh_leader_backoff"]) fetchMetadataRetries, _ := strconv.Atoi(rawConfig["fetch_metadata_retries"]) fetchMetadataBackoff, _ := time.ParseDuration(rawConfig["fetch_metadata_backoff"]) time.ParseDuration(rawConfig["fetch_metadata_backoff"]) offsetsCommitMaxRetries, _ := strconv.Atoi(rawConfig["offsets_commit_max_retries"]) flushInterval, _ := time.ParseDuration(rawConfig["flush_interval"]) deploymentTimeout, _ := time.ParseDuration(rawConfig["deployment_timeout"]) zkConfig := kafkaClient.NewZookeeperConfig() zkConfig.ZookeeperConnect = []string{rawConfig["zookeeper_connect"]} zkConfig.ZookeeperTimeout = zkTimeout config := kafkaClient.DefaultConsumerConfig() config.Groupid = rawConfig["group_id"] config.NumWorkers = numWorkers config.MaxWorkerRetries = maxWorkerRetries config.WorkerBackoff = workerBackoff config.WorkerRetryThreshold = int32(workerRetryThreshold) config.WorkerThresholdTimeWindow = workerConsideredFailedTimeWindow config.WorkerTaskTimeout = workerTaskTimeout config.WorkerManagersStopTimeout = workerManagersStopTimeout config.BarrierTimeout = rebalanceBarrierTimeout config.RebalanceMaxRetries = int32(rebalanceMaxRetries) config.RebalanceBackoff = rebalanceBackoff config.PartitionAssignmentStrategy = partitionAssignmentStrategy config.ExcludeInternalTopics = excludeInternalTopics config.NumConsumerFetchers = numConsumerFetchers config.FetchBatchSize = fetchBatchSize config.FetchMessageMaxBytes = int32(fetchMessageMaxBytes) config.FetchMinBytes = int32(fetchMinBytes) config.FetchBatchTimeout = fetchBatchTimeout config.FetchTopicMetadataRetries = fetchMetadataRetries config.FetchTopicMetadataBackoff = fetchMetadataBackoff config.RequeueAskNextBackoff = requeueAskNextBackoff config.FetchWaitMaxMs = int32(fetchWaitMaxMs) config.SocketTimeout = socketTimeout config.QueuedMaxMessages = int32(queuedMaxMessages) config.RefreshLeaderBackoff = refreshLeaderBackoff config.Coordinator = kafkaClient.NewZookeeperCoordinator(zkConfig) config.OffsetsStorage = rawConfig["offsets_storage"] config.AutoOffsetReset = rawConfig["auto_offset_reset"] config.OffsetsCommitMaxRetries = offsetsCommitMaxRetries config.DeploymentTimeout = deploymentTimeout return config, rawConfig["topic"], numConsumers, rawConfig["graphite_connect"], flushInterval }