func produceAvro() { config1 := kafka.DefaultProducerConfig() config1.BrokerList = strings.Split(*brokerList, ",") config1.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry) config1.AckSuccesses = true producer1 := kafka.NewSaramaProducer(config1) config2 := kafka.DefaultProducerConfig() config2.BrokerList = strings.Split(*brokerList, ",") config2.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry) producer2 := kafka.NewSaramaProducer(config2) avroSchema, err := avro.ParseSchemaFile(*avroSchema) if err != nil { panic(err) } _, err = kafka.NewCachedSchemaRegistryClient(*schemaRegistry).Register(avroSchema.GetName()+"-value", avroSchema) if err != nil { panic(err) } decoder := kafka.NewKafkaAvroDecoder(*schemaRegistry) go func() { for message := range producer1.Successes() { rawRecord, err := decoder.Decode(message.Value.([]byte)) if err != nil { panic(err) } record := rawRecord.(*avro.GenericRecord) timings := record.Get("timings").([]interface{}) timings = append(timings, time.Now().UnixNano()/int64(time.Millisecond)) record.Set("timings", timings) producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: record} } }() for _ = range time.Tick(1 * time.Second) { messagesSent := 0 for messagesSent < *perSecond { record := avro.NewGenericRecord(avroSchema) record.Set("id", int64(0)) record.Set("timings", []int64{time.Now().UnixNano() / int64(time.Millisecond)}) record.Set("value", []byte{}) message := &kafka.ProducerMessage{Topic: *topic1, Value: record} producer1.Input() <- message messagesSent++ } } }
func produceLogLineProtobuf() { config1 := kafka.DefaultProducerConfig() config1.BrokerList = strings.Split(*brokerList, ",") config1.AckSuccesses = true producer1 := kafka.NewSaramaProducer(config1) config2 := kafka.DefaultProducerConfig() config2.BrokerList = strings.Split(*brokerList, ",") producer2 := kafka.NewSaramaProducer(config2) go func() { for message := range producer1.Successes() { line := &sp.LogLine{} proto.Unmarshal(message.Value.([]byte), line) line.Timings = append(line.Timings, time.Now().UnixNano()/int64(time.Millisecond)) bytes, err := proto.Marshal(line) if err != nil { panic(err) } producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: bytes} } }() for _ = range time.Tick(1 * time.Second) { messagesSent := 0 for messagesSent < *perSecond { line := &sp.LogLine{} line.Line = proto.String("") line.Timings = []int64{time.Now().UnixNano() / int64(time.Millisecond)} bytes, err := proto.Marshal(line) if err != nil { panic(err) } message := &kafka.ProducerMessage{Topic: *topic1, Value: bytes} producer1.Input() <- message messagesSent++ } } }
func main() { parseAndValidateArgs() ctrlc := make(chan os.Signal, 1) signal.Notify(ctrlc, os.Interrupt) producerConfig := kafka.DefaultProducerConfig() producerConfig.BrokerList = strings.Split(*brokerList, ",") zkConfig := kafka.NewZookeeperConfig() zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",") coordinator := kafka.NewZookeeperCoordinator(zkConfig) config := kafka.DefaultConsumerConfig() config.Debug = true config.Groupid = "perf-mirror" config.AutoOffsetReset = "smallest" config.Coordinator = coordinator config.WorkerFailedAttemptCallback = FailedAttemptCallback config.WorkerFailureCallback = FailedCallback if *siesta { config.LowLevelClient = kafka.NewSiestaClient(config) } if protobuf { setupProtoConfig(config) } else { producerConfig.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry) setupAvroConfig(config) } producer = kafka.NewSaramaProducer(producerConfig) consumer := kafka.NewConsumer(config) go consumer.StartStatic(map[string]int{*consumeTopic: 1}) <-ctrlc fmt.Println("Shutdown triggered, closing consumer") <-consumer.Close() producer.Close() }