Exemplo n.º 1
0
func produceAvro() {
	config1 := kafka.DefaultProducerConfig()
	config1.BrokerList = strings.Split(*brokerList, ",")
	config1.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
	config1.AckSuccesses = true

	producer1 := kafka.NewSaramaProducer(config1)

	config2 := kafka.DefaultProducerConfig()
	config2.BrokerList = strings.Split(*brokerList, ",")
	config2.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
	producer2 := kafka.NewSaramaProducer(config2)

	avroSchema, err := avro.ParseSchemaFile(*avroSchema)
	if err != nil {
		panic(err)
	}

	_, err = kafka.NewCachedSchemaRegistryClient(*schemaRegistry).Register(avroSchema.GetName()+"-value", avroSchema)
	if err != nil {
		panic(err)
	}

	decoder := kafka.NewKafkaAvroDecoder(*schemaRegistry)
	go func() {
		for message := range producer1.Successes() {
			rawRecord, err := decoder.Decode(message.Value.([]byte))
			if err != nil {
				panic(err)
			}
			record := rawRecord.(*avro.GenericRecord)
			timings := record.Get("timings").([]interface{})
			timings = append(timings, time.Now().UnixNano()/int64(time.Millisecond))
			record.Set("timings", timings)

			producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: record}
		}
	}()

	for _ = range time.Tick(1 * time.Second) {
		messagesSent := 0
		for messagesSent < *perSecond {
			record := avro.NewGenericRecord(avroSchema)
			record.Set("id", int64(0))
			record.Set("timings", []int64{time.Now().UnixNano() / int64(time.Millisecond)})
			record.Set("value", []byte{})

			message := &kafka.ProducerMessage{Topic: *topic1, Value: record}
			producer1.Input() <- message
			messagesSent++
		}
	}
}
Exemplo n.º 2
0
func parseAndValidateArgs() *kafka.MirrorMakerConfig {
	flag.Var(&consumerConfig, "consumer.config", "Path to consumer configuration file.")
	flag.Parse()
	runtime.GOMAXPROCS(*maxProcs)

	if (*whitelist != "" && *blacklist != "") || (*whitelist == "" && *blacklist == "") {
		fmt.Println("Exactly one of whitelist or blacklist is required.")
		os.Exit(1)
	}
	if *producerConfig == "" {
		fmt.Println("Producer config is required.")
		os.Exit(1)
	}
	if len(consumerConfig) == 0 {
		fmt.Println("At least one consumer config is required.")
		os.Exit(1)
	}
	if *queueSize < 0 {
		fmt.Println("Queue size should be equal or greater than 0")
		os.Exit(1)
	}
	if *timingsProducerConfig == "" && *schemaRegistryUrl == "" {
		fmt.Println("--schema.registry.url parameter is required when --timings is used")
	}

	config := kafka.NewMirrorMakerConfig()
	config.Blacklist = *blacklist
	config.Whitelist = *whitelist
	config.ChannelSize = *queueSize
	config.ConsumerConfigs = []string(consumerConfig)
	config.NumProducers = *numProducers
	config.NumStreams = *numStreams
	config.PreservePartitions = *preservePartitions
	config.PreserveOrder = *preserveOrder
	config.ProducerConfig = *producerConfig
	config.TopicPrefix = *prefix
	if *schemaRegistryUrl != "" {
		config.KeyEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistryUrl)
		config.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistryUrl)
		config.KeyDecoder = kafka.NewKafkaAvroDecoder(*schemaRegistryUrl)
		config.ValueDecoder = kafka.NewKafkaAvroDecoder(*schemaRegistryUrl)
	}
	config.TimingsProducerConfig = *timingsProducerConfig

	return config
}
Exemplo n.º 3
0
func main() {
	parseAndValidateArgs()
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	producerConfig := kafka.DefaultProducerConfig()
	producerConfig.BrokerList = strings.Split(*brokerList, ",")

	zkConfig := kafka.NewZookeeperConfig()
	zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",")
	coordinator := kafka.NewZookeeperCoordinator(zkConfig)

	config := kafka.DefaultConsumerConfig()
	config.Debug = true
	config.Groupid = "perf-mirror"
	config.AutoOffsetReset = "smallest"
	config.Coordinator = coordinator
	config.WorkerFailedAttemptCallback = FailedAttemptCallback
	config.WorkerFailureCallback = FailedCallback
	if *siesta {
		config.LowLevelClient = kafka.NewSiestaClient(config)
	}

	if protobuf {
		setupProtoConfig(config)
	} else {
		producerConfig.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
		setupAvroConfig(config)
	}

	producer = kafka.NewSaramaProducer(producerConfig)
	consumer := kafka.NewConsumer(config)

	go consumer.StartStatic(map[string]int{*consumeTopic: 1})

	<-ctrlc
	fmt.Println("Shutdown triggered, closing consumer")
	<-consumer.Close()
	producer.Close()
}