func (this *EventFetcher) createConsumer() (*kafka.Consumer, error) { fmt.Println(this.config.ZkConnect) coordinatorConfig := kafka.NewZookeeperConfig() coordinatorConfig.ZookeeperConnect = []string{this.config.ZkConnect} coordinator := kafka.NewZookeeperCoordinator(coordinatorConfig) consumerConfig := kafka.DefaultConsumerConfig() consumerConfig.AutoOffsetReset = kafka.LargestOffset consumerConfig.Coordinator = coordinator consumerConfig.Groupid = "event-dashboard" consumerConfig.ValueDecoder = kafka.NewKafkaAvroDecoder(this.config.SchemaRegistryUrl) consumerConfig.WorkerFailureCallback = func(_ *kafka.WorkerManager) kafka.FailedDecision { return kafka.CommitOffsetAndContinue } consumerConfig.WorkerFailedAttemptCallback = func(_ *kafka.Task, _ kafka.WorkerResult) kafka.FailedDecision { return kafka.CommitOffsetAndContinue } consumerConfig.Strategy = func(_ *kafka.Worker, msg *kafka.Message, taskId kafka.TaskId) kafka.WorkerResult { if record, ok := msg.DecodedValue.(*avro.GenericRecord); ok { this.events <- &Event{ EventName: record.Get("eventname").(string), Second: record.Get("second").(int64), Framework: record.Get("framework").(string), Latency: record.Get("latency").(int64), ReceivedCount: record.Get("received_count").(int64), SentCount: record.Get("sent_count").(int64), } } else { return kafka.NewProcessingFailedResult(taskId) } return kafka.NewSuccessfulResult(taskId) } consumer, err := kafka.NewSlaveConsumer(consumerConfig) return consumer, err }
func (this *EventFetcher) createConsumer() *kafka.Consumer { coordinatorConfig := kafka.NewZookeeperConfig() coordinatorConfig.ZookeeperConnect = []string{this.config.ZkConnect} coordinator := kafka.NewZookeeperCoordinator(coordinatorConfig) consumerConfig := kafka.DefaultConsumerConfig() consumerConfig.AutoOffsetReset = kafka.LargestOffset consumerConfig.Coordinator = coordinator consumerConfig.Groupid = "event-dashboard" consumerConfig.ValueDecoder = kafka.NewKafkaAvroDecoder(this.config.SchemaRegistryUrl) consumerConfig.WorkerFailureCallback = func(_ *kafka.WorkerManager) kafka.FailedDecision { return kafka.CommitOffsetAndContinue } consumerConfig.WorkerFailedAttemptCallback = func(_ *kafka.Task, _ kafka.WorkerResult) kafka.FailedDecision { return kafka.CommitOffsetAndContinue } consumerConfig.Strategy = func(_ *kafka.Worker, msg *kafka.Message, taskId kafka.TaskId) kafka.WorkerResult { if record, ok := msg.DecodedValue.(*avro.GenericRecord); ok { this.events <- &Event{ Topic: record.Get("topic").(string), ConsumerId: record.Get("consumerid").(string), Partition: record.Get("partition").(string), EventName: record.Get("eventname").(string), Second: record.Get("second").(int64), Operation: record.Get("operation").(string), Value: record.Get("value").(int64), Cnt: record.Get("cnt").(int64), } } else { return kafka.NewProcessingFailedResult(taskId) } return kafka.NewSuccessfulResult(taskId) } return kafka.NewConsumer(consumerConfig) }