Example #1
0
func newProducerFromZookeeper() (sarama.Client, sarama.SyncProducer, error) {
	brokers, err := kafkazk.LookupBrokers(cfg.zkstring)
	if err != nil {
		return nil, nil, err
	}

	brokerStr := make([]string, len(brokers))
	for i, b := range brokers {
		brokerStr[i] = fmt.Sprintf("%s:%d", b.Host, b.Port)
	}

	logger.Println("connecting to Kafka, using brokers from ZooKeeper:", brokerStr)
	client, err := sarama.NewClient(brokerStr, sarama.NewConfig())
	if err != nil {
		return nil, nil, err
	}

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		return nil, nil, err
	}

	return client, producer, nil
}
Example #2
0
func main() {
	kingpin.Parse()

	if *server == "" {
		fmt.Fprintln(os.Stderr, "no zookeeper given")
		os.Exit(1)
	}

	brokers, err := kafkazk.LookupBrokers(*server)
	if err != nil {
		log.Fatal(err)
	}

	brokerString := make([]string, len(brokers))
	for i, b := range brokers {
		brokerString[i] = fmt.Sprintf("%s:%d", b.Host, b.Port)
	}

	if *verbose {
		log.Println("connecting to kafka, using brokers from zookeeper:", brokerString)
	}

	clientConfig := sarama.NewConfig()
	clientConfig.ClientID = *clientID
	client, err := sarama.NewClient(brokerString, clientConfig)
	if err != nil {
		log.Fatal(err)
	}

	defer client.Close()

	if *verbose {
		log.Println("connected")
	}

	topics, err := client.Topics()
	if err != nil {
		log.Fatal(err)
	}

	sort.Strings(topics)

	if *list {
		for _, topic := range topics {
			fmt.Println(topic)
		}
		os.Exit(0)
	}

	if *topic == "" {
		log.Fatal("no topic given")
	}

	index := sort.SearchStrings(topics, *topic)
	found := index < len(topics) && topics[index] == *topic
	if !found {
		log.Fatal("no such topic")
		os.Exit(1)
	}

	if *latest {
		latestOffset, err := client.GetOffset(*topic, int32(*partition), sarama.OffsetNewest)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Println(latestOffset)
		os.Exit(0)
	}

	master, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		log.Fatal(err)
	}

	if *verbose {
		log.Println("master consumer ready")
	}

	consumeFrom := sarama.OffsetNewest
	if *fromBeginning {
		consumeFrom = sarama.OffsetOldest
	}
	if *offset > 0 {
		consumeFrom = *offset
	}

	consumer, err := master.ConsumePartition(*topic, int32(*partition), consumeFrom)
	if err != nil {
		log.Fatal(err)
	}

	if *verbose {
		log.Println("partition consumer ready")
	}
	defer consumer.Close()

	msgCount := 0

	defer func() {
		if *verbose {
			log.Println("got", msgCount, "message(s)")
		}
	}()

	stop := make(chan os.Signal, 1)
	signal.Notify(stop, os.Interrupt)

	for {
		select {
		case error := <-consumer.Errors():
			log.Println(error)
		case message := <-consumer.Messages():
			if *verbose {
				log.Println("offset", message.Offset)
			}
			fmt.Println(string(message.Value[:]))
			msgCount++
			if *max != 0 && msgCount >= *max {
				return
			}
		case <-stop:
			if *verbose {
				log.Println("stopping")
			}
			return
		}
	}
}