Beispiel #1
0
func produceAvro() {
	config1 := kafka.DefaultProducerConfig()
	config1.BrokerList = strings.Split(*brokerList, ",")
	config1.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
	config1.AckSuccesses = true

	producer1 := kafka.NewSaramaProducer(config1)

	config2 := kafka.DefaultProducerConfig()
	config2.BrokerList = strings.Split(*brokerList, ",")
	config2.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
	producer2 := kafka.NewSaramaProducer(config2)

	avroSchema, err := avro.ParseSchemaFile(*avroSchema)
	if err != nil {
		panic(err)
	}

	_, err = kafka.NewCachedSchemaRegistryClient(*schemaRegistry).Register(avroSchema.GetName()+"-value", avroSchema)
	if err != nil {
		panic(err)
	}

	decoder := kafka.NewKafkaAvroDecoder(*schemaRegistry)
	go func() {
		for message := range producer1.Successes() {
			rawRecord, err := decoder.Decode(message.Value.([]byte))
			if err != nil {
				panic(err)
			}
			record := rawRecord.(*avro.GenericRecord)
			timings := record.Get("timings").([]interface{})
			timings = append(timings, time.Now().UnixNano()/int64(time.Millisecond))
			record.Set("timings", timings)

			producer2.Input() <- &kafka.ProducerMessage{Topic: *topic2, Value: record}
		}
	}()

	for _ = range time.Tick(1 * time.Second) {
		messagesSent := 0
		for messagesSent < *perSecond {
			record := avro.NewGenericRecord(avroSchema)
			record.Set("id", int64(0))
			record.Set("timings", []int64{time.Now().UnixNano() / int64(time.Millisecond)})
			record.Set("value", []byte{})

			message := &kafka.ProducerMessage{Topic: *topic1, Value: record}
			producer1.Input() <- message
			messagesSent++
		}
	}
}
func parseAndValidateArgs() *kafka.MirrorMakerConfig {
	flag.Var(&consumerConfig, "consumer.config", "Path to consumer configuration file.")
	flag.Parse()
	runtime.GOMAXPROCS(*maxProcs)

	if (*whitelist != "" && *blacklist != "") || (*whitelist == "" && *blacklist == "") {
		fmt.Println("Exactly one of whitelist or blacklist is required.")
		os.Exit(1)
	}
	if *producerConfig == "" {
		fmt.Println("Producer config is required.")
		os.Exit(1)
	}
	if len(consumerConfig) == 0 {
		fmt.Println("At least one consumer config is required.")
		os.Exit(1)
	}
	if *queueSize < 0 {
		fmt.Println("Queue size should be equal or greater than 0")
		os.Exit(1)
	}
	if *timingsProducerConfig == "" && *schemaRegistryUrl == "" {
		fmt.Println("--schema.registry.url parameter is required when --timings is used")
	}

	config := kafka.NewMirrorMakerConfig()
	config.Blacklist = *blacklist
	config.Whitelist = *whitelist
	config.ChannelSize = *queueSize
	config.ConsumerConfigs = []string(consumerConfig)
	config.NumProducers = *numProducers
	config.NumStreams = *numStreams
	config.PreservePartitions = *preservePartitions
	config.PreserveOrder = *preserveOrder
	config.ProducerConfig = *producerConfig
	config.TopicPrefix = *prefix
	if *schemaRegistryUrl != "" {
		config.KeyEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistryUrl)
		config.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistryUrl)
		config.KeyDecoder = kafka.NewKafkaAvroDecoder(*schemaRegistryUrl)
		config.ValueDecoder = kafka.NewKafkaAvroDecoder(*schemaRegistryUrl)
	}
	config.TimingsProducerConfig = *timingsProducerConfig

	return config
}
Beispiel #3
0
func (e *Executor) serializer(transform string) func(interface{}) ([]byte, error) {
	switch transform {
	case TransformNone:
		return siesta.StringSerializer
	case TransformAvro:
		return go_kafka_client.NewKafkaAvroEncoder(Config.SchemaRegistryUrl).Encode
	}

	// should not happen
	panic("Unknown transformation type")
}
Beispiel #4
0
func main() {
	parseAndValidateArgs()
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	producerConfig := kafka.DefaultProducerConfig()
	producerConfig.BrokerList = strings.Split(*brokerList, ",")

	zkConfig := kafka.NewZookeeperConfig()
	zkConfig.ZookeeperConnect = strings.Split(*zookeeper, ",")
	coordinator := kafka.NewZookeeperCoordinator(zkConfig)

	config := kafka.DefaultConsumerConfig()
	config.Debug = true
	config.Groupid = "perf-mirror"
	config.AutoOffsetReset = "smallest"
	config.Coordinator = coordinator
	config.WorkerFailedAttemptCallback = FailedAttemptCallback
	config.WorkerFailureCallback = FailedCallback
	if *siesta {
		config.LowLevelClient = kafka.NewSiestaClient(config)
	}

	if protobuf {
		setupProtoConfig(config)
	} else {
		producerConfig.ValueEncoder = kafka.NewKafkaAvroEncoder(*schemaRegistry)
		setupAvroConfig(config)
	}

	producer = kafka.NewSaramaProducer(producerConfig)
	consumer := kafka.NewConsumer(config)

	go consumer.StartStatic(map[string]int{*consumeTopic: 1})

	<-ctrlc
	fmt.Println("Shutdown triggered, closing consumer")
	<-consumer.Close()
	producer.Close()
}
Beispiel #5
0
func (this *TransformExecutor) startProducer() {
	producerConfig, err := kafka.ProducerConfigFromFile(this.config.ProducerConfig)
	if err != nil {
		panic(err)
	}

	cfgMap := make(map[string]string)
	err = cfg.Load(this.config.ProducerConfig, cfgMap)
	if err != nil {
		panic(err)
	}

	this.avroDecoder = kafka.NewKafkaAvroDecoder(cfgMap["schema.registry.url"])

	producerConfig.KeyEncoder = kafka.NewKafkaAvroEncoder(cfgMap["schema.registry.url"])
	producerConfig.ValueEncoder = producerConfig.KeyEncoder
	producerConfig.SendBufferSize = 10000
	producerConfig.BatchSize = 2000
	producerConfig.MaxMessagesPerRequest = 5000

	this.producer = kafka.NewSaramaProducer(producerConfig)
	go this.produceRoutine()
}
Beispiel #6
0
	"os/signal"
	"time"
)

var readTopic string
var writeTopic string
var group = "ping-pong-go-group"

var broker = "localhost:9092"
var zookeeper = "localhost:2181"
var schemaRepo = "http://localhost:8081"

var kafkaProducer *producer.KafkaProducer = nil
var kafkaConsumer *go_kafka_client.Consumer = nil

var encoder = go_kafka_client.NewKafkaAvroEncoder(schemaRepo)

func main() {
	parseArgs()

	go_kafka_client.Logger = go_kafka_client.NewDefaultLogger(go_kafka_client.ErrorLevel)
	kafkaProducer = producer.NewKafkaProducer(writeTopic, []string{broker})

	//Coordinator settings
	zookeeperConfig := go_kafka_client.NewZookeeperConfig()
	zookeeperConfig.ZookeeperConnect = []string{zookeeper}

	//Actual consumer settings
	consumerConfig := go_kafka_client.DefaultConsumerConfig()
	consumerConfig.AutoOffsetReset = go_kafka_client.SmallestOffset
	consumerConfig.Coordinator = go_kafka_client.NewZookeeperCoordinator(zookeeperConfig)