Beispiel #1
0
func partitionExpectationConsumer(pc sarama.PartitionConsumer, expectations <-chan *sarama.ProducerMessage, wg *sync.WaitGroup) {
	defer wg.Done()
	defer func() {
		if err := pc.Close(); err != nil {
			logger.Println("Failed to close partitionconsumer:", err)
		}
	}()

	for expectation := range expectations {
		msg := <-pc.Messages()

		if msg.Offset != expectation.Offset {
			fmt.Printf("Unexpected offset %d!\n", msg.Offset)
		}

		key, _ := expectation.Key.Encode()
		if string(key) != string(msg.Key) {
			fmt.Printf("Unexpected key: %v!\n", msg.Key)
		}
		if msg.Value != nil {
			fmt.Printf("Unexpected value: %v!\n", msg.Value)
		}

		stats.LogConsumed(expectation)
	}
}
Beispiel #2
0
func consumePartitionLoop(
	closer chan struct{},
	out chan string,
	pc sarama.PartitionConsumer,
	p int32,
	end int64,
) {
	for {
		timeout := make(<-chan time.Time)
		if config.consume.timeout > 0 {
			timeout = time.After(config.consume.timeout)
		}

		select {
		case <-timeout:
			log.Printf("Consuming from partition [%v] timed out.", p)
			pc.Close()
			return
		case <-closer:
			pc.Close()
			return
		case msg, ok := <-pc.Messages():
			if ok {
				m := consumedMessage{
					Partition: msg.Partition,
					Offset:    msg.Offset,
				}
				k := string(msg.Key)
				if msg.Key != nil {
					m.Key = &k
				}
				v := string(msg.Value)
				if msg.Value != nil {
					m.Value = &v
				}

				byts, err := json.Marshal(m)
				if err != nil {
					fmt.Fprintf(os.Stderr, "Quitting due to unexpected error during marshal: %v\n", err)
					close(closer)
					return
				}
				out <- string(byts)
			}
			if end > 0 && msg.Offset >= end {
				pc.Close()
				return
			}
		}
	}
}