Example #1
0
func produceAvro() {
	config1 := kafka.DefaultProducerConfig()
	config1.BrokerList = strings.Split(*brokerList, ",")
	config1.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
	config1.AckSuccesses = true

	producer1 := kafka.NewSaramaProducer(config1)

	config2 := kafka.DefaultProducerConfig()
	config2.BrokerList = strings.Split(*brokerList, ",")
	config2.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
	producer2 := kafka.NewSaramaProducer(config2)

	avroSchema, err := avro.ParseSchemaFile(*avroSchema)
	if err != nil {
		panic(err)
	}

	_, err = kafka.NewCachedSchemaRegistryClient(*schemaRegistry).Register(avroSchema.GetName()+"-value", avroSchema)
	if err != nil {
		panic(err)
	}

	decoder := kafka.NewKafkaAvroDecoder(*schemaRegistry)
	go func() {
		for message := range producer1.Successes() {
			rawRecord, err := decoder.Decode(message.Value.([]byte))
			if err != nil {
				panic(err)
			}
			record := rawRecord.(*avro.GenericRecord)
			timings := record.Get("timings").([]interface{})
			timings = append(timings, time.Now().UnixNano()/int64(time.Millisecond))
			record.Set("timings", timings)

			producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: record}
		}
	}()

	for _ = range time.Tick(1 * time.Second) {
		messagesSent := 0
		for messagesSent < *perSecond {
			record := avro.NewGenericRecord(avroSchema)
			record.Set("id", int64(0))
			record.Set("timings", []int64{time.Now().UnixNano() / int64(time.Millisecond)})
			record.Set("value", []byte{})

			message := &kafka.ProducerMessage{Topic: *topic1, Value: record}
			producer1.Input() <- message
			messagesSent++
		}
	}
}
Example #2
0
func produceLogLineProtobuf() {
	config1 := kafka.DefaultProducerConfig()
	config1.BrokerList = strings.Split(*brokerList, ",")
	config1.AckSuccesses = true
	producer1 := kafka.NewSaramaProducer(config1)

	config2 := kafka.DefaultProducerConfig()
	config2.BrokerList = strings.Split(*brokerList, ",")
	producer2 := kafka.NewSaramaProducer(config2)

	go func() {
		for message := range producer1.Successes() {
			line := &sp.LogLine{}
			proto.Unmarshal(message.Value.([]byte), line)
			line.Timings = append(line.Timings, time.Now().UnixNano()/int64(time.Millisecond))
			bytes, err := proto.Marshal(line)
			if err != nil {
				panic(err)
			}

			producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: bytes}
		}
	}()

	for _ = range time.Tick(1 * time.Second) {
		messagesSent := 0
		for messagesSent < *perSecond {
			line := &sp.LogLine{}
			line.Line = proto.String("")
			line.Timings = []int64{time.Now().UnixNano() / int64(time.Millisecond)}
			bytes, err := proto.Marshal(line)
			if err != nil {
				panic(err)
			}

			message := &kafka.ProducerMessage{Topic: *topic1, Value: bytes}
			producer1.Input() <- message
			messagesSent++
		}
	}
}
Example #3
0
func main() {
	parseAndValidateArgs()
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	producerConfig := kafka.DefaultProducerConfig()
	producerConfig.BrokerList = strings.Split(*brokerList, ",")

	zkConfig := kafka.NewZookeeperConfig()
	zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",")
	coordinator := kafka.NewZookeeperCoordinator(zkConfig)

	config := kafka.DefaultConsumerConfig()
	config.Debug = true
	config.Groupid = "perf-mirror"
	config.AutoOffsetReset = "smallest"
	config.Coordinator = coordinator
	config.WorkerFailedAttemptCallback = FailedAttemptCallback
	config.WorkerFailureCallback = FailedCallback
	if *siesta {
		config.LowLevelClient = kafka.NewSiestaClient(config)
	}

	if protobuf {
		setupProtoConfig(config)
	} else {
		producerConfig.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
		setupAvroConfig(config)
	}

	producer = kafka.NewSaramaProducer(producerConfig)
	consumer := kafka.NewConsumer(config)

	go consumer.StartStatic(map[string]int{*consumeTopic: 1})

	<-ctrlc
	fmt.Println("Shutdown triggered, closing consumer")
	<-consumer.Close()
	producer.Close()
}
Example #4
0
func (this *TransformExecutor) startProducer() {
	producerConfig, err := kafka.ProducerConfigFromFile(this.config.ProducerConfig)
	if err != nil {
		panic(err)
	}

	cfgMap := make(map[string]string)
	err = cfg.Load(this.config.ProducerConfig, cfgMap)
	if err != nil {
		panic(err)
	}

	this.avroDecoder = kafka.NewKafkaAvroDecoder(cfgMap["schema.registry.url"])

	producerConfig.KeyEncoder = kafka.NewKafkaAvroEncoder(cfgMap["schema.registry.url"])
	producerConfig.ValueEncoder = producerConfig.KeyEncoder
	producerConfig.SendBufferSize = 10000
	producerConfig.BatchSize = 2000
	producerConfig.MaxMessagesPerRequest = 5000

	this.producer = kafka.NewSaramaProducer(producerConfig)
	go this.produceRoutine()
}