func main() {
log := logging.MustGetLogger("MarshalExample")
// Construct the marshaler. There will be one of these globally, and it's thread safe so can
// be used from any goroutine.
marshaler, err := marshal.NewMarshaler(
"marshal_example_client_id",
"marshal_example_consumer_group_id",
[]string{"127.0.0.1:9092"})
if err != nil {
log.Fatalf("Failed to construct marshaler: %s", err)
}
// Make sure to terminate the Marshaler. This ensures that we release all of the partition
// locks we're holding so other consumers can pick them up.
defer marshaler.Terminate()
// Now we set up a basic consumer; and we enable GreedyClaims which is useful in low QPS
// environments as it will cause the consumer to claim as many partitions as it can
// up front. Of course, if you have a very busy topic with many partitions, you will
// not want to use this.
options := marshal.NewConsumerOptions()
options.GreedyClaims = true
consumer, err := marshaler.NewConsumer([]string{"some-topic"}, options)
if err != nil {
log.Fatalf("Failed to construct consumer: %s", err)
}
defer consumer.Terminate(true)
// Now we can get the consumption channel. Messages will be available in this channel
// and you can consume from it in many different goroutines if your message processing
// is such that it takes a while.
msgChan := consumer.ConsumeChannel()
// You can spin up many goroutines to process messages; how many depends entirely on the type
// of workload you have. Note that if you turn StrictOrdering on, spinning up multiple
// routines may not help as much as you expect. See the docs.
for i := 0; i < 10; i++ {
i := i
go func() {
for {
msg := <-msgChan
log.Info("[%d] got message: %s", i, msg.Value)
// Now we have to commit the message now that we're done with it. If you don't
// commit, then Marshal will never record forward progress and will eventually
// terminate.
consumer.Commit(msg)
}
}()
}
}
func main() {
broker := flag.String("broker", "localhost:9092", "ip:port of a single broker")
group := flag.String("group", "debug-group", "group ID to use")
client := flag.String("client", "debug-client", "client ID to use")
topic := flag.String("topic", "test64", "topic to test against")
claimTopic := flag.Bool("claim-topic", false, "claim entire topic mode")
greedyClaim := flag.Bool("greedy-claim", false, "turn on greedy claims")
fastReclaim := flag.Bool("fast-reclaim", false, "enable fast reclaim mode")
printOnly := flag.Bool("print-state-only", false, "only print state, do not claim")
flag.Parse()
// Raise marshal debugging level
logging.SetLevel(logging.DEBUG, "KafkaMarshal")
// Construction timing
var m *marshal.Marshaler
timeIt("construct Marshaler", func() {
var err error
m, err = marshal.NewMarshaler(*client, *group, []string{*broker})
if err != nil {
log.Fatalf("Failed to construct Marshaler: %s", err)
}
})
defer timeIt("terminate Marshaler", func() { m.Terminate() })
// If we're in print mode just do that and exit
if *printOnly {
m.PrintState()
return
}
// Ensure target topic exists
partitions := m.Partitions(*topic)
if partitions == 0 {
log.Fatalf("Topic %s has no partitions/does not exist.", *topic)
}
log.Info("Topic %s has %d partitions.", *topic, partitions)
// Set up consumption of the topic with the options they gave us
options := marshal.NewConsumerOptions()
options.GreedyClaims = *greedyClaim
options.FastReclaim = *fastReclaim
options.ClaimEntireTopic = *claimTopic
timeIt("claim all partitions", func() {
var c *marshal.Consumer
timeIt("construct Consumer", func() {
var err error
c, err = m.NewConsumer([]string{*topic}, options)
if err != nil {
log.Fatalf("Failed to construct consumer: %s", err)
}
})
defer timeIt("terminate Consumer", func() { c.Terminate(false) })
// Wait for all partitions to be claimed
for c.GetCurrentLoad() < partitions {
time.Sleep(10 * time.Millisecond)
}
})
m.PrintState()
}