Esempio n. 1
0
func NewKafkaAvroAdapter(route *router.Route) (router.LogAdapter, error) {
	brokers := readBrokers(route.Address)
	if len(brokers) == 0 {
		return nil, errorf("The Kafka broker host:port is missing. Did you specify it as a route address?")
	}

	topic := readTopic(route.Address, route.Options)
	if topic == "" {
		return nil, errorf("The Kafka topic is missing. Did you specify it as a route option?")
	}

	schemaUrl := readSchemaRegistryUrl(route.Options)
	if schemaUrl == "" {
		return nil, errorf("The schema registry url is missing. Did you specify it as a route option?")
	}

	registry := kafkaavro.NewKafkaAvroEncoder(schemaUrl)

	var schema avro.Schema
	schema, err := avro.ParseSchema(messageSchema)
	if err != nil {
		return nil, errorf("The schema could not be parsed")
	}

	if os.Getenv("DEBUG") != "" {
		log.Printf("Starting Kafka producer for address: %s, topic: %s.\n", brokers, topic)
	}

	var retries int
	retries, err = strconv.Atoi(os.Getenv("KAFKA_CONNECT_RETRIES"))
	if err != nil {
		retries = 3
	}
	var producer sarama.AsyncProducer
	for i := 0; i < retries; i++ {
		producer, err = sarama.NewAsyncProducer(brokers, newConfig())
		if err != nil {
			if os.Getenv("DEBUG") != "" {
				log.Println("Couldn't create Kafka producer. Retrying...", err)
			}
			if i == retries-1 {
				return nil, errorf("Couldn't create Kafka producer. %v", err)
			}
		} else {
			time.Sleep(1 * time.Second)
		}
	}

	return &KafkaAvroAdapter{
		route:    route,
		brokers:  brokers,
		topic:    topic,
		registry: registry,
		schema:   schema,
		producer: producer,
	}, nil
}
Esempio n. 2
0
func NewCodahaleKafkaReporter(topic string, schemaRegistryUrl string, producerConfig *producer.ProducerConfig, connectorConfig *siesta.ConnectorConfig) (*CodahaleKafkaReporter, error) {
	encoder := kafkaavro.NewKafkaAvroEncoder(schemaRegistryUrl)
	connector, err := siesta.NewDefaultConnector(connectorConfig)
	if err != nil {
		return nil, err
	}

	return &CodahaleKafkaReporter{
		topic:    topic,
		producer: producer.NewKafkaProducer(producerConfig, producer.ByteSerializer, encoder.Encode, connector),
	}, nil
}
Esempio n. 3
0
// NewKafkaLogEmitter creates a new KafkaLogEmitter with a provided configuration.
func NewKafkaLogEmitter(config *KafkaLogEmitterConfig) (*KafkaLogEmitter, error) {
	encoder := kafkaavro.NewKafkaAvroEncoder(config.SchemaRegistryUrl)
	connector, err := siesta.NewDefaultConnector(config.ConnectorConfig)
	if err != nil {
		return nil, err
	}
	emitter := &KafkaLogEmitter{
		config:   config,
		producer: producer.NewKafkaProducer(config.ProducerConfig, producer.ByteSerializer, encoder.Encode, connector),
	}

	return emitter, nil
}
Esempio n. 4
0
func validate() *syslog.SyslogProducerConfig {
	if brokerList == "" {
		fmt.Println("broker.list is required.")
		os.Exit(1)
	}

	if topic == "" {
		fmt.Println("Topic is required.")
		os.Exit(1)
	}

	if sendAvro && schemaRegistryUrl == "" {
		fmt.Println("Schema Registry URL is required for --avro flag")
		os.Exit(1)
	}

	config := syslog.NewSyslogProducerConfig()
	config.ProducerConfig = producer.NewProducerConfig()
	config.ProducerConfig.RequiredAcks = requiredAcks
	config.ProducerConfig.AckTimeoutMs = int32(acksTimeout)
	config.BrokerList = brokerList
	config.NumProducers = numProducers
	config.Topic = topic
	config.TCPAddr = fmt.Sprintf("%s:%d", tcpHost, tcpPort)
	config.UDPAddr = fmt.Sprintf("%s:%d", udpHost, udpPort)

	if sendAvro {
		serializer := kafkaavro.NewKafkaAvroEncoder(schemaRegistryUrl)
		config.ValueSerializer = serializer.Encode
		config.Transformer = avroTransformer
	}

	if sendProtobuf {
		config.ValueSerializer = producer.ByteSerializer
		config.Transformer = protobufTransformer
	}

	return config
}