func (this *EventFetcher) createConsumer() (*kafka.Consumer, error) {
fmt.Println(this.config.ZkConnect)
coordinatorConfig := kafka.NewZookeeperConfig()
coordinatorConfig.ZookeeperConnect = []string{this.config.ZkConnect}
coordinator := kafka.NewZookeeperCoordinator(coordinatorConfig)
consumerConfig := kafka.DefaultConsumerConfig()
consumerConfig.AutoOffsetReset = kafka.LargestOffset
consumerConfig.Coordinator = coordinator
consumerConfig.Groupid = "event-dashboard"
consumerConfig.ValueDecoder = kafka.NewKafkaAvroDecoder(this.config.SchemaRegistryUrl)
consumerConfig.WorkerFailureCallback = func(_ *kafka.WorkerManager) kafka.FailedDecision {
return kafka.CommitOffsetAndContinue
}
consumerConfig.WorkerFailedAttemptCallback = func(_ *kafka.Task, _ kafka.WorkerResult) kafka.FailedDecision {
return kafka.CommitOffsetAndContinue
}
consumerConfig.Strategy = func(_ *kafka.Worker, msg *kafka.Message, taskId kafka.TaskId) kafka.WorkerResult {
if record, ok := msg.DecodedValue.(*avro.GenericRecord); ok {
this.events <- &Event{
EventName: record.Get("eventname").(string),
Second: record.Get("second").(int64),
Framework: record.Get("framework").(string),
Latency: record.Get("latency").(int64),
ReceivedCount: record.Get("received_count").(int64),
SentCount: record.Get("sent_count").(int64),
}
} else {
return kafka.NewProcessingFailedResult(taskId)
}
return kafka.NewSuccessfulResult(taskId)
}
consumer, err := kafka.NewSlaveConsumer(consumerConfig)
return consumer, err
}
func (this *EventFetcher) createConsumer() *kafka.Consumer {
coordinatorConfig := kafka.NewZookeeperConfig()
coordinatorConfig.ZookeeperConnect = []string{this.config.ZkConnect}
coordinator := kafka.NewZookeeperCoordinator(coordinatorConfig)
consumerConfig := kafka.DefaultConsumerConfig()
consumerConfig.AutoOffsetReset = kafka.LargestOffset
consumerConfig.Coordinator = coordinator
consumerConfig.Groupid = "event-dashboard"
consumerConfig.ValueDecoder = kafka.NewKafkaAvroDecoder(this.config.SchemaRegistryUrl)
consumerConfig.WorkerFailureCallback = func(_ *kafka.WorkerManager) kafka.FailedDecision {
return kafka.CommitOffsetAndContinue
}
consumerConfig.WorkerFailedAttemptCallback = func(_ *kafka.Task, _ kafka.WorkerResult) kafka.FailedDecision {
return kafka.CommitOffsetAndContinue
}
consumerConfig.Strategy = func(_ *kafka.Worker, msg *kafka.Message, taskId kafka.TaskId) kafka.WorkerResult {
if record, ok := msg.DecodedValue.(*avro.GenericRecord); ok {
this.events <- &Event{
Topic: record.Get("topic").(string),
ConsumerId: record.Get("consumerid").(string),
Partition: record.Get("partition").(string),
EventName: record.Get("eventname").(string),
Second: record.Get("second").(int64),
Operation: record.Get("operation").(string),
Value: record.Get("value").(int64),
Cnt: record.Get("cnt").(int64),
}
} else {
return kafka.NewProcessingFailedResult(taskId)
}
return kafka.NewSuccessfulResult(taskId)
}
return kafka.NewConsumer(consumerConfig)
}