func produceAvro() {
config1 := kafka.DefaultProducerConfig()
config1.BrokerList = strings.Split(*brokerList, ",")
config1.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
config1.AckSuccesses = true
producer1 := kafka.NewSaramaProducer(config1)
config2 := kafka.DefaultProducerConfig()
config2.BrokerList = strings.Split(*brokerList, ",")
config2.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
producer2 := kafka.NewSaramaProducer(config2)
avroSchema, err := avro.ParseSchemaFile(*avroSchema)
if err != nil {
panic(err)
}
_, err = kafka.NewCachedSchemaRegistryClient(*schemaRegistry).Register(avroSchema.GetName()+"-value", avroSchema)
if err != nil {
panic(err)
}
decoder := kafka.NewKafkaAvroDecoder(*schemaRegistry)
go func() {
for message := range producer1.Successes() {
rawRecord, err := decoder.Decode(message.Value.([]byte))
if err != nil {
panic(err)
}
record := rawRecord.(*avro.GenericRecord)
timings := record.Get("timings").([]interface{})
timings = append(timings, time.Now().UnixNano()/int64(time.Millisecond))
record.Set("timings", timings)
producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: record}
}
}()
for _ = range time.Tick(1 * time.Second) {
messagesSent := 0
for messagesSent < *perSecond {
record := avro.NewGenericRecord(avroSchema)
record.Set("id", int64(0))
record.Set("timings", []int64{time.Now().UnixNano() / int64(time.Millisecond)})
record.Set("value", []byte{})
message := &kafka.ProducerMessage{Topic: *topic1, Value: record}
producer1.Input() <- message
messagesSent++
}
}
}
func parseAndValidateArgs() *kafka.MirrorMakerConfig {
flag.Var(&consumerConfig, "consumer.config", "Path to consumer configuration file.")
flag.Parse()
runtime.GOMAXPROCS(*maxProcs)
if (*whitelist != "" && *blacklist != "") || (*whitelist == "" && *blacklist == "") {
fmt.Println("Exactly one of whitelist or blacklist is required.")
os.Exit(1)
}
if *producerConfig == "" {
fmt.Println("Producer config is required.")
os.Exit(1)
}
if len(consumerConfig) == 0 {
fmt.Println("At least one consumer config is required.")
os.Exit(1)
}
if *queueSize < 0 {
fmt.Println("Queue size should be equal or greater than 0")
os.Exit(1)
}
if *timingsProducerConfig == "" && *schemaRegistryUrl == "" {
fmt.Println("--schema.registry.url parameter is required when --timings is used")
}
config := kafka.NewMirrorMakerConfig()
config.Blacklist = *blacklist
config.Whitelist = *whitelist
config.ChannelSize = *queueSize
config.ConsumerConfigs = []string(consumerConfig)
config.NumProducers = *numProducers
config.NumStreams = *numStreams
config.PreservePartitions = *preservePartitions
config.PreserveOrder = *preserveOrder
config.ProducerConfig = *producerConfig
config.TopicPrefix = *prefix
if *schemaRegistryUrl != "" {
config.KeyEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistryUrl)
config.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistryUrl)
config.KeyDecoder = kafka.NewKafkaAvroDecoder(*schemaRegistryUrl)
config.ValueDecoder = kafka.NewKafkaAvroDecoder(*schemaRegistryUrl)
}
config.TimingsProducerConfig = *timingsProducerConfig
return config
}
func (e *Executor) serializer(transform string) func(interface{}) ([]byte, error) {
switch transform {
case TransformNone:
return siesta.StringSerializer
case TransformAvro:
return go_kafka_client.NewKafkaAvroEncoder(Config.SchemaRegistryUrl).Encode
}
// should not happen
panic("Unknown transformation type")
}
func main() {
parseAndValidateArgs()
ctrlc := make(chan os.Signal, 1)
signal.Notify(ctrlc, os.Interrupt)
producerConfig := kafka.DefaultProducerConfig()
producerConfig.BrokerList = strings.Split(*brokerList, ",")
zkConfig := kafka.NewZookeeperConfig()
zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",")
coordinator := kafka.NewZookeeperCoordinator(zkConfig)
config := kafka.DefaultConsumerConfig()
config.Debug = true
config.Groupid = "perf-mirror"
config.AutoOffsetReset = "smallest"
config.Coordinator = coordinator
config.WorkerFailedAttemptCallback = FailedAttemptCallback
config.WorkerFailureCallback = FailedCallback
if *siesta {
config.LowLevelClient = kafka.NewSiestaClient(config)
}
if protobuf {
setupProtoConfig(config)
} else {
producerConfig.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
setupAvroConfig(config)
}
producer = kafka.NewSaramaProducer(producerConfig)
consumer := kafka.NewConsumer(config)
go consumer.StartStatic(map[string]int{*consumeTopic: 1})
<-ctrlc
fmt.Println("Shutdown triggered, closing consumer")
<-consumer.Close()
producer.Close()
}
func (this *TransformExecutor) startProducer() {
producerConfig, err := kafka.ProducerConfigFromFile(this.config.ProducerConfig)
if err != nil {
panic(err)
}
cfgMap := make(map[string]string)
err = cfg.Load(this.config.ProducerConfig, cfgMap)
if err != nil {
panic(err)
}
this.avroDecoder = kafka.NewKafkaAvroDecoder(cfgMap["schema.registry.url"])
producerConfig.KeyEncoder = kafka.NewKafkaAvroEncoder(cfgMap["schema.registry.url"])
producerConfig.ValueEncoder = producerConfig.KeyEncoder
producerConfig.SendBufferSize = 10000
producerConfig.BatchSize = 2000
producerConfig.MaxMessagesPerRequest = 5000
this.producer = kafka.NewSaramaProducer(producerConfig)
go this.produceRoutine()
}
"os/signal"
"time"
)
var readTopic string
var writeTopic string
var group = "ping-pong-go-group"
var broker = "localhost:9092"
var zookeeper = "localhost:2181"
var schemaRepo = "http://localhost:8081"
var kafkaProducer *producer.KafkaProducer = nil
var kafkaConsumer *go_kafka_client.Consumer = nil
var encoder = go_kafka_client.NewKafkaAvroEncoder(schemaRepo)
func main() {
parseArgs()
go_kafka_client.Logger = go_kafka_client.NewDefaultLogger(go_kafka_client.ErrorLevel)
kafkaProducer = producer.NewKafkaProducer(writeTopic, []string{broker})
//Coordinator settings
zookeeperConfig := go_kafka_client.NewZookeeperConfig()
zookeeperConfig.ZookeeperConnect = []string{zookeeper}
//Actual consumer settings
consumerConfig := go_kafka_client.DefaultConsumerConfig()
consumerConfig.AutoOffsetReset = go_kafka_client.SmallestOffset
consumerConfig.Coordinator = go_kafka_client.NewZookeeperCoordinator(zookeeperConfig)