func produceAvro() {
config1 := kafka.DefaultProducerConfig()
config1.BrokerList = strings.Split(*brokerList, ",")
config1.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
config1.AckSuccesses = true
producer1 := kafka.NewSaramaProducer(config1)
config2 := kafka.DefaultProducerConfig()
config2.BrokerList = strings.Split(*brokerList, ",")
config2.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
producer2 := kafka.NewSaramaProducer(config2)
avroSchema, err := avro.ParseSchemaFile(*avroSchema)
if err != nil {
panic(err)
}
_, err = kafka.NewCachedSchemaRegistryClient(*schemaRegistry).Register(avroSchema.GetName()+"-value", avroSchema)
if err != nil {
panic(err)
}
decoder := kafka.NewKafkaAvroDecoder(*schemaRegistry)
go func() {
for message := range producer1.Successes() {
rawRecord, err := decoder.Decode(message.Value.([]byte))
if err != nil {
panic(err)
}
record := rawRecord.(*avro.GenericRecord)
timings := record.Get("timings").([]interface{})
timings = append(timings, time.Now().UnixNano()/int64(time.Millisecond))
record.Set("timings", timings)
producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: record}
}
}()
for _ = range time.Tick(1 * time.Second) {
messagesSent := 0
for messagesSent < *perSecond {
record := avro.NewGenericRecord(avroSchema)
record.Set("id", int64(0))
record.Set("timings", []int64{time.Now().UnixNano() / int64(time.Millisecond)})
record.Set("value", []byte{})
message := &kafka.ProducerMessage{Topic: *topic1, Value: record}
producer1.Input() <- message
messagesSent++
}
}
}
func produceLogLineProtobuf() {
config1 := kafka.DefaultProducerConfig()
config1.BrokerList = strings.Split(*brokerList, ",")
config1.AckSuccesses = true
producer1 := kafka.NewSaramaProducer(config1)
config2 := kafka.DefaultProducerConfig()
config2.BrokerList = strings.Split(*brokerList, ",")
producer2 := kafka.NewSaramaProducer(config2)
go func() {
for message := range producer1.Successes() {
line := &sp.LogLine{}
proto.Unmarshal(message.Value.([]byte), line)
line.Timings = append(line.Timings, time.Now().UnixNano()/int64(time.Millisecond))
bytes, err := proto.Marshal(line)
if err != nil {
panic(err)
}
producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: bytes}
}
}()
for _ = range time.Tick(1 * time.Second) {
messagesSent := 0
for messagesSent < *perSecond {
line := &sp.LogLine{}
line.Line = proto.String("")
line.Timings = []int64{time.Now().UnixNano() / int64(time.Millisecond)}
bytes, err := proto.Marshal(line)
if err != nil {
panic(err)
}
message := &kafka.ProducerMessage{Topic: *topic1, Value: bytes}
producer1.Input() <- message
messagesSent++
}
}
}
func main() {
parseAndValidateArgs()
ctrlc := make(chan os.Signal, 1)
signal.Notify(ctrlc, os.Interrupt)
producerConfig := kafka.DefaultProducerConfig()
producerConfig.BrokerList = strings.Split(*brokerList, ",")
zkConfig := kafka.NewZookeeperConfig()
zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",")
coordinator := kafka.NewZookeeperCoordinator(zkConfig)
config := kafka.DefaultConsumerConfig()
config.Debug = true
config.Groupid = "perf-mirror"
config.AutoOffsetReset = "smallest"
config.Coordinator = coordinator
config.WorkerFailedAttemptCallback = FailedAttemptCallback
config.WorkerFailureCallback = FailedCallback
if *siesta {
config.LowLevelClient = kafka.NewSiestaClient(config)
}
if protobuf {
setupProtoConfig(config)
} else {
producerConfig.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
setupAvroConfig(config)
}
producer = kafka.NewSaramaProducer(producerConfig)
consumer := kafka.NewConsumer(config)
go consumer.StartStatic(map[string]int{*consumeTopic: 1})
<-ctrlc
fmt.Println("Shutdown triggered, closing consumer")
<-consumer.Close()
producer.Close()
}
func (this *TransformExecutor) startProducer() {
producerConfig, err := kafka.ProducerConfigFromFile(this.config.ProducerConfig)
if err != nil {
panic(err)
}
cfgMap := make(map[string]string)
err = cfg.Load(this.config.ProducerConfig, cfgMap)
if err != nil {
panic(err)
}
this.avroDecoder = kafka.NewKafkaAvroDecoder(cfgMap["schema.registry.url"])
producerConfig.KeyEncoder = kafka.NewKafkaAvroEncoder(cfgMap["schema.registry.url"])
producerConfig.ValueEncoder = producerConfig.KeyEncoder
producerConfig.SendBufferSize = 10000
producerConfig.BatchSize = 2000
producerConfig.MaxMessagesPerRequest = 5000
this.producer = kafka.NewSaramaProducer(producerConfig)
go this.produceRoutine()
}