コード例 #1
0
ファイル: produce.go プロジェクト: tulios/kafka-revolver
func main() {
	configFile := flag.String("c", "", "Config file")
	messageValue := flag.String("m", "", "Message")
	amount := flag.Int("a", 1, "Amount of messages")
	flag.Parse()

	if *configFile == "" || *messageValue == "" {
		flag.PrintDefaults()
		os.Exit(1)
	}

	options, err := revolver.LoadOptions(*configFile)
	if err != nil {
		log.Fatalln(err)
		os.Exit(1)
	}

	sarama.Logger = logger

	var keyEncoder, valueEncoder sarama.Encoder

	keyEncoder = sarama.StringEncoder(time.Now().String())
	if *messageValue != "" {
		valueEncoder = sarama.StringEncoder(*messageValue)
	}

	config := sarama.NewConfig()
	config.Producer.Partitioner = sarama.NewRandomPartitioner
	producer, err := sarama.NewSyncProducer(options.Brokers, config)
	if err != nil {
		logger.Fatalln("FAILED to open the producer:", err)
	}
	defer producer.Close()
	topic := options.KafkaTopics[0]

	for i := 0; i < *amount; i++ {
		partition, offset, err := producer.SendMessage(&sarama.ProducerMessage{
			Topic: topic,
			Key:   keyEncoder,
			Value: valueEncoder,
		})

		if err != nil {
			logger.Println("FAILED to produce message:", err)

		} else {
			logger.Printf("msg: %d, topic=%s\tpartition=%d\toffset=%d\n", i, topic, partition, offset)
		}
	}
}
コード例 #2
0
ファイル: main.go プロジェクト: tulios/kafka-revolver
func main() {
	cmdOpts := readCmdOptions()
	options, err := revolver.LoadOptions(cmdOpts.configFile)
	if err != nil {
		log.Fatalln(err)
		flag.PrintDefaults()
		os.Exit(1)
	}

	sarama.Logger = log.New(os.Stdout, "[kafka] ", log.LstdFlags)
	log.Printf("Kafka brokers: %s", strings.Join(options.Brokers, ", "))

	consumer, err := revolver.NewConsumer(options)
	if err != nil {
		log.Fatalln(err)
		os.Exit(1)
	}

	setupTrapInterrupt(consumer)
	setupOnErrorLog(consumer)

	offsets := make(map[string]map[int32]int64)

	for message := range consumer.Messages() {
		if offsets[message.Topic] == nil {
			offsets[message.Topic] = make(map[int32]int64)
		}

		partition := offsets[message.Topic][message.Partition]
		if partition != 0 && partition != message.Offset-1 {
			log.Printf(
				"Unexpected offset on %s:%d. Expected %d, found %d, diff %d.\n",
				message.Topic,
				message.Partition,
				partition+1,
				message.Offset,
				message.Offset-partition+1,
			)
		}

		if revolver.Consume(message, options.ConsumersCmd) {
			offsets[message.Topic][message.Partition] = message.Offset
			consumer.CommitUpto(message)
		}
	}
}