func produceAvro() {
config1 := kafka.DefaultProducerConfig()
config1.BrokerList = strings.Split(*brokerList, ",")
config1.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
config1.AckSuccesses = true
producer1 := kafka.NewSaramaProducer(config1)
config2 := kafka.DefaultProducerConfig()
config2.BrokerList = strings.Split(*brokerList, ",")
config2.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
producer2 := kafka.NewSaramaProducer(config2)
avroSchema, err := avro.ParseSchemaFile(*avroSchema)
if err != nil {
panic(err)
}
_, err = kafka.NewCachedSchemaRegistryClient(*schemaRegistry).Register(avroSchema.GetName()+"-value", avroSchema)
if err != nil {
panic(err)
}
decoder := kafka.NewKafkaAvroDecoder(*schemaRegistry)
go func() {
for message := range producer1.Successes() {
rawRecord, err := decoder.Decode(message.Value.([]byte))
if err != nil {
panic(err)
}
record := rawRecord.(*avro.GenericRecord)
timings := record.Get("timings").([]interface{})
timings = append(timings, time.Now().UnixNano()/int64(time.Millisecond))
record.Set("timings", timings)
producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: record}
}
}()
for _ = range time.Tick(1 * time.Second) {
messagesSent := 0
for messagesSent < *perSecond {
record := avro.NewGenericRecord(avroSchema)
record.Set("id", int64(0))
record.Set("timings", []int64{time.Now().UnixNano() / int64(time.Millisecond)})
record.Set("value", []byte{})
message := &kafka.ProducerMessage{Topic: *topic1, Value: record}
producer1.Input() <- message
messagesSent++
}
}
}
func parseAndValidateArgs() *kafka.MirrorMakerConfig {
flag.Var(&consumerConfig, "consumer.config", "Path to consumer configuration file.")
flag.Parse()
runtime.GOMAXPROCS(*maxProcs)
if (*whitelist != "" && *blacklist != "") || (*whitelist == "" && *blacklist == "") {
fmt.Println("Exactly one of whitelist or blacklist is required.")
os.Exit(1)
}
if *producerConfig == "" {
fmt.Println("Producer config is required.")
os.Exit(1)
}
if len(consumerConfig) == 0 {
fmt.Println("At least one consumer config is required.")
os.Exit(1)
}
if *queueSize < 0 {
fmt.Println("Queue size should be equal or greater than 0")
os.Exit(1)
}
if *timingsProducerConfig == "" && *schemaRegistryUrl == "" {
fmt.Println("--schema.registry.url parameter is required when --timings is used")
}
config := kafka.NewMirrorMakerConfig()
config.Blacklist = *blacklist
config.Whitelist = *whitelist
config.ChannelSize = *queueSize
config.ConsumerConfigs = []string(consumerConfig)
config.NumProducers = *numProducers
config.NumStreams = *numStreams
config.PreservePartitions = *preservePartitions
config.PreserveOrder = *preserveOrder
config.ProducerConfig = *producerConfig
config.TopicPrefix = *prefix
if *schemaRegistryUrl != "" {
config.KeyEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistryUrl)
config.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistryUrl)
config.KeyDecoder = kafka.NewKafkaAvroDecoder(*schemaRegistryUrl)
config.ValueDecoder = kafka.NewKafkaAvroDecoder(*schemaRegistryUrl)
}
config.TimingsProducerConfig = *timingsProducerConfig
return config
}
func main() {
parseAndValidateArgs()
ctrlc := make(chan os.Signal, 1)
signal.Notify(ctrlc, os.Interrupt)
producerConfig := kafka.DefaultProducerConfig()
producerConfig.BrokerList = strings.Split(*brokerList, ",")
zkConfig := kafka.NewZookeeperConfig()
zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",")
coordinator := kafka.NewZookeeperCoordinator(zkConfig)
config := kafka.DefaultConsumerConfig()
config.Debug = true
config.Groupid = "perf-mirror"
config.AutoOffsetReset = "smallest"
config.Coordinator = coordinator
config.WorkerFailedAttemptCallback = FailedAttemptCallback
config.WorkerFailureCallback = FailedCallback
if *siesta {
config.LowLevelClient = kafka.NewSiestaClient(config)
}
if protobuf {
setupProtoConfig(config)
} else {
producerConfig.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
setupAvroConfig(config)
}
producer = kafka.NewSaramaProducer(producerConfig)
consumer := kafka.NewConsumer(config)
go consumer.StartStatic(map[string]int{*consumeTopic: 1})
<-ctrlc
fmt.Println("Shutdown triggered, closing consumer")
<-consumer.Close()
producer.Close()
}