func startNewConsumer(config kafkaClient.ConsumerConfig, topic string) *kafkaClient.Consumer {
config.Strategy = GetStrategy(config.Consumerid)
config.WorkerFailureCallback = FailedCallback
config.WorkerFailedAttemptCallback = FailedAttemptCallback
consumer := kafkaClient.NewConsumer(&config)
topics := map[string]int{topic: config.NumConsumerFetchers}
go func() {
consumer.StartStatic(topics)
}()
return consumer
}
func main() {
parseAndValidateArgs()
ctrlc := make(chan os.Signal, 1)
signal.Notify(ctrlc, os.Interrupt)
producerConfig := kafka.DefaultProducerConfig()
producerConfig.BrokerList = strings.Split(*brokerList, ",")
zkConfig := kafka.NewZookeeperConfig()
zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",")
coordinator := kafka.NewZookeeperCoordinator(zkConfig)
config := kafka.DefaultConsumerConfig()
config.Debug = true
config.Groupid = "perf-mirror"
config.AutoOffsetReset = "smallest"
config.Coordinator = coordinator
config.WorkerFailedAttemptCallback = FailedAttemptCallback
config.WorkerFailureCallback = FailedCallback
if *siesta {
config.LowLevelClient = kafka.NewSiestaClient(config)
}
if protobuf {
setupProtoConfig(config)
} else {
producerConfig.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
setupAvroConfig(config)
}
producer = kafka.NewSaramaProducer(producerConfig)
consumer := kafka.NewConsumer(config)
go consumer.StartStatic(map[string]int{*consumeTopic: 1})
<-ctrlc
fmt.Println("Shutdown triggered, closing consumer")
<-consumer.Close()
producer.Close()
}
func main() {
parseAndValidateArgs()
ctrlc := make(chan os.Signal, 1)
signal.Notify(ctrlc, os.Interrupt)
zkConfig := kafka.NewZookeeperConfig()
zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",")
coordinator := kafka.NewZookeeperCoordinator(zkConfig)
config := kafka.DefaultConsumerConfig()
config.Groupid = "perf-consumer"
config.AutoOffsetReset = "smallest"
config.Coordinator = coordinator
config.WorkerFailedAttemptCallback = FailedAttemptCallback
config.WorkerFailureCallback = FailedCallback
if *siesta {
config.LowLevelClient = kafka.NewSiestaClient(config)
}
if protobuf {
setupLogLineProtoConfig(config)
} else {
setupAvroConfig(config)
}
consumer := kafka.NewConsumer(config)
go consumer.StartStatic(map[string]int{*topic: 2})
go func() {
latencies := make([]metrics.Histogram, 0)
endToEnd := metrics.NewRegisteredHistogram(fmt.Sprint("Latency-end-to-end"), metrics.DefaultRegistry, metrics.NewUniformSample(10000))
go func() {
for {
time.Sleep(1 * time.Second)
for i, meter := range latencies {
fmt.Printf("Step %d: %f\n", i+1, meter.Mean())
}
fmt.Printf("End-to-end: %f\n", endToEnd.Mean())
fmt.Println()
}
}()
initialized := false
for timing := range timings {
if !initialized {
for i := 1; i < len(timing); i++ {
latencies = append(latencies, metrics.NewRegisteredHistogram(fmt.Sprintf("Latency-step-%d", i), metrics.DefaultRegistry, metrics.NewUniformSample(10000)))
}
initialized = true
}
if len(timing)-1 != len(latencies) {
fmt.Println("Got wrong latencies, skipping..")
continue
}
for i := 1; i < len(timing); i++ {
latencies[i-1].Update(int64(timing[i] - timing[i-1]))
}
endToEnd.Update(int64(timing[len(timing)-1] - timing[0]))
}
}()
<-ctrlc
fmt.Println("Shutdown triggered, closing consumer")
<-consumer.Close()
close(timings)
}