Example #1
0
func partitionExpectationConsumer(pc sarama.PartitionConsumer, expectations <-chan *sarama.ProducerMessage, wg *sync.WaitGroup) {
	defer wg.Done()
	defer func() {
		if err := pc.Close(); err != nil {
			logger.Println("Failed to close partitionconsumer:", err)
		}
	}()

	for expectation := range expectations {
		msg := <-pc.Messages()

		if msg.Offset != expectation.Offset {
			fmt.Printf("Unexpected offset %d!\n", msg.Offset)
		}

		key, _ := expectation.Key.Encode()
		if string(key) != string(msg.Key) {
			fmt.Printf("Unexpected key: %v!\n", msg.Key)
		}
		if msg.Value != nil {
			fmt.Printf("Unexpected value: %v!\n", msg.Value)
		}

		stats.LogConsumed(expectation)
	}
}
Example #2
0
func (suite *KafkaTester) Test01() {
	t := suite.T()
	assert := assert.New(t)

	const M1 = "message one"
	const M2 = "message two"

	var producer sarama.AsyncProducer
	var consumer sarama.Consumer
	var partitionConsumer sarama.PartitionConsumer

	var err error

	topic := makeTopicName()

	{
		config := sarama.NewConfig()
		config.Producer.Return.Successes = false
		config.Producer.Return.Errors = false

		producer, err = sarama.NewAsyncProducer([]string{suite.server}, config)
		assert.NoError(err)
		defer close(t, producer)

		producer.Input() <- &sarama.ProducerMessage{
			Topic: topic,
			Key:   nil,
			Value: sarama.StringEncoder(M1)}

		producer.Input() <- &sarama.ProducerMessage{
			Topic: topic,
			Key:   nil,
			Value: sarama.StringEncoder(M2)}
	}

	{
		consumer, err = sarama.NewConsumer([]string{suite.server}, nil)
		assert.NoError(err)
		defer close(t, consumer)

		partitionConsumer, err = consumer.ConsumePartition(topic, 0, 0)
		assert.NoError(err)
		defer close(t, partitionConsumer)
	}

	{
		mssg1 := <-partitionConsumer.Messages()
		//t.Logf("Consumed: offset:%d  value:%v", mssg1.Offset, string(mssg1.Value))
		mssg2 := <-partitionConsumer.Messages()
		//t.Logf("Consumed: offset:%d  value:%v", mssg2.Offset, string(mssg2.Value))

		assert.EqualValues(M1, string(mssg1.Value))
		assert.EqualValues(M2, string(mssg2.Value))
	}
}
Example #3
0
func handleConsumer(consumer kafka.PartitionConsumer) {
	for message := range consumer.Messages() {
		log.V(2).Printf("Got message: %v\n", message)
		nextRoundRobinClient().inbox <- messages.Message{
			Value:        message.Value,
			Partition:    message.Partition,
			Offset:       message.Offset,
			IsAckRequest: false,
		}
	}
}
Example #4
0
// Consume kafka messages
func consumeMessages(producer kafka.AsyncProducer, consumer kafka.PartitionConsumer) error {
	select {
	case message := <-consumer.Messages():
		var dat map[string]interface{}
		if err := json.Unmarshal(message.Value, &dat); err != nil {
			fmt.Printf("Error with Unmarshal: %s\n", err)
		} else {
			monascaTS, monascaUS, monascaSS := createMonascaSpec(dat)
			mtm, mum, msm := marshalMonasca(monascaTS, monascaUS, monascaSS)

			produceMonascaMetrics(producer, dat, mtm, mum, msm)

		}
	case err := <-consumer.Errors():
		return err
	}
	return nil
}
Example #5
0
func consumePartitionLoop(
	closer chan struct{},
	out chan string,
	pc sarama.PartitionConsumer,
	p int32,
	end int64,
) {
	for {
		timeout := make(<-chan time.Time)
		if config.consume.timeout > 0 {
			timeout = time.After(config.consume.timeout)
		}

		select {
		case <-timeout:
			log.Printf("Consuming from partition [%v] timed out.", p)
			pc.Close()
			return
		case <-closer:
			pc.Close()
			return
		case msg, ok := <-pc.Messages():
			if ok {
				m := consumedMessage{
					Partition: msg.Partition,
					Offset:    msg.Offset,
				}
				k := string(msg.Key)
				if msg.Key != nil {
					m.Key = &k
				}
				v := string(msg.Value)
				if msg.Value != nil {
					m.Value = &v
				}

				byts, err := json.Marshal(m)
				if err != nil {
					fmt.Fprintf(os.Stderr, "Quitting due to unexpected error during marshal: %v\n", err)
					close(closer)
					return
				}
				out <- string(byts)
			}
			if end > 0 && msg.Offset >= end {
				pc.Close()
				return
			}
		}
	}
}