Esempio n. 1
0
func main() {
	log := logging.MustGetLogger("MarshalExample")

	// Construct the marshaler. There will be one of these globally, and it's thread safe so can
	// be used from any goroutine.
	marshaler, err := marshal.NewMarshaler(
		"marshal_example_client_id",
		"marshal_example_consumer_group_id",
		[]string{"127.0.0.1:9092"})
	if err != nil {
		log.Fatalf("Failed to construct marshaler: %s", err)
	}

	// Make sure to terminate the Marshaler. This ensures that we release all of the partition
	// locks we're holding so other consumers can pick them up.
	defer marshaler.Terminate()

	// Now we set up a basic consumer; and we enable GreedyClaims which is useful in low QPS
	// environments as it will cause the consumer to claim as many partitions as it can
	// up front. Of course, if you have a very busy topic with many partitions, you will
	// not want to use this.
	options := marshal.NewConsumerOptions()
	options.GreedyClaims = true

	consumer, err := marshaler.NewConsumer([]string{"some-topic"}, options)
	if err != nil {
		log.Fatalf("Failed to construct consumer: %s", err)
	}
	defer consumer.Terminate(true)

	// Now we can get the consumption channel. Messages will be available in this channel
	// and you can consume from it in many different goroutines if your message processing
	// is such that it takes a while.
	msgChan := consumer.ConsumeChannel()

	// You can spin up many goroutines to process messages; how many depends entirely on the type
	// of workload you have. Note that if you turn StrictOrdering on, spinning up multiple
	// routines may not help as much as you expect. See the docs.
	for i := 0; i < 10; i++ {
		i := i
		go func() {
			for {
				msg := <-msgChan
				log.Info("[%d] got message: %s", i, msg.Value)

				// Now we have to commit the message now that we're done with it. If you don't
				// commit, then Marshal will never record forward progress and will eventually
				// terminate.
				consumer.Commit(msg)
			}
		}()
	}
}
Esempio n. 2
0
func main() {
	broker := flag.String("broker", "localhost:9092", "ip:port of a single broker")
	group := flag.String("group", "debug-group", "group ID to use")
	client := flag.String("client", "debug-client", "client ID to use")
	topic := flag.String("topic", "test64", "topic to test against")
	claimTopic := flag.Bool("claim-topic", false, "claim entire topic mode")
	greedyClaim := flag.Bool("greedy-claim", false, "turn on greedy claims")
	fastReclaim := flag.Bool("fast-reclaim", false, "enable fast reclaim mode")
	printOnly := flag.Bool("print-state-only", false, "only print state, do not claim")
	flag.Parse()

	// Raise marshal debugging level
	logging.SetLevel(logging.DEBUG, "KafkaMarshal")

	// Construction timing
	var m *marshal.Marshaler
	timeIt("construct Marshaler", func() {
		var err error
		m, err = marshal.NewMarshaler(*client, *group, []string{*broker})
		if err != nil {
			log.Fatalf("Failed to construct Marshaler: %s", err)
		}
	})
	defer timeIt("terminate Marshaler", func() { m.Terminate() })

	// If we're in print mode just do that and exit
	if *printOnly {
		m.PrintState()
		return
	}

	// Ensure target topic exists
	partitions := m.Partitions(*topic)
	if partitions == 0 {
		log.Fatalf("Topic %s has no partitions/does not exist.", *topic)
	}
	log.Info("Topic %s has %d partitions.", *topic, partitions)

	// Set up consumption of the topic with the options they gave us
	options := marshal.NewConsumerOptions()
	options.GreedyClaims = *greedyClaim
	options.FastReclaim = *fastReclaim
	options.ClaimEntireTopic = *claimTopic

	timeIt("claim all partitions", func() {
		var c *marshal.Consumer
		timeIt("construct Consumer", func() {
			var err error
			c, err = m.NewConsumer([]string{*topic}, options)
			if err != nil {
				log.Fatalf("Failed to construct consumer: %s", err)
			}
		})
		defer timeIt("terminate Consumer", func() { c.Terminate(false) })

		// Wait for all partitions to be claimed
		for c.GetCurrentLoad() < partitions {
			time.Sleep(10 * time.Millisecond)
		}
	})

	m.PrintState()
}