func tailPartitions(client *sarama.Client, topic string, partitions []int32) {
var wg sync.WaitGroup
wg.Add(len(partitions))
tailConsumer := func(partition int32) {
defer wg.Done()
consumerConfig := sarama.NewConsumerConfig()
consumerConfig.OffsetMethod = sarama.OffsetMethodManual
consumerConfig.OffsetValue = offset
consumer, err := sarama.NewConsumer(client, topic, partition, "", consumerConfig)
if err != nil {
logger.Fatalf("err creating consumer: %s", err)
}
defer consumer.Close()
for event := range consumer.Events() {
logger.Printf("partition=%d offset=%d key=%s value=%s", event.Partition, event.Offset, event.Key, event.Value)
}
}
for _, partition := range partitions {
go tailConsumer(partition)
}
wg.Wait()
}
func NewKafka(numberOfMessages int, testLatency bool) *Kafka {
pubClient, _ := sarama.NewClient("pub", []string{"localhost:9092"}, sarama.NewClientConfig())
subClient, _ := sarama.NewClient("sub", []string{"localhost:9092"}, sarama.NewClientConfig())
topic := "test"
pub, _ := sarama.NewProducer(pubClient, sarama.NewProducerConfig())
consumerConfig := sarama.NewConsumerConfig()
consumerConfig.OffsetMethod = sarama.OffsetMethodNewest // Only read new messages
consumerConfig.DefaultFetchSize = 10 * 1024 * 1024
sub, _ := sarama.NewConsumer(subClient, topic, 0, "test", consumerConfig)
var handler benchmark.MessageHandler
if testLatency {
handler = &benchmark.LatencyMessageHandler{
NumberOfMessages: numberOfMessages,
Latencies: []float32{},
}
} else {
handler = &benchmark.ThroughputMessageHandler{NumberOfMessages: numberOfMessages}
}
return &Kafka{
handler: handler,
pubClient: pubClient,
subClient: subClient,
pub: pub,
sub: sub,
topic: topic,
}
}
func main() {
client, err := kafka.NewClient("my_client", []string{"localhost:9092"}, nil)
if err != nil {
panic(err)
} else {
fmt.Println("> connected")
}
defer client.Close()
consumer, err := kafka.NewConsumer(client, "my_topic", 0, "my_consumer_group", kafka.NewConsumerConfig())
if err != nil {
panic(err)
} else {
fmt.Println("> consumer ready")
}
defer consumer.Close()
msgCount := 0
consumerLoop:
for {
select {
case event := <-consumer.Events():
if event.Err != nil {
panic(event.Err)
}
msgCount++
case <-time.After(5 * time.Second):
fmt.Println("> timed out")
break consumerLoop
}
}
fmt.Println("Got", msgCount, "messages.")
}
func main() {
err := envconfig.Process("KC", &appConfig)
if err != nil {
log.Fatal(err.Error())
}
fmt.Println(appConfig)
client, err := sarama.NewClient("my_client", []string{appConfig.Broker}, nil)
if err != nil {
panic(err)
} else {
fmt.Println("> connected")
}
defer client.Close()
consumerConfig := sarama.NewConsumerConfig()
consumerConfig.OffsetMethod = sarama.OffsetMethodNewest
consumer, err := sarama.NewConsumer(client, appConfig.Topic,
appConfig.Partition, "my_consumer_group", consumerConfig)
if err != nil {
panic(err)
} else {
fmt.Println("> consumer ready")
}
defer consumer.Close()
var event *sarama.ConsumerEvent
for {
select {
case event = <-consumer.Events():
if event.Err != nil {
panic(event.Err)
}
fmt.Println("got message:", string(event.Value))
}
}
}
func Consume(quit chan bool, server []string, topic, cursor string, partition int32, updater func(Key, Value, Type string, Offset int64) bool) {
logPrintln("Start Consume")
cursorFile, errc := os.OpenFile(cursor, os.O_CREATE|os.O_RDWR, 0666)
if errc != nil {
panic(errc)
}
defer cursorFile.Close()
client, err := sarama.NewClient("crontab_client", server, nil)
if err != nil {
panic(err)
} else {
logPrintln("kafka consumer connected")
}
defer client.Close()
cfg := sarama.NewConsumerConfig()
cfg.OffsetMethod = sarama.OffsetMethodManual
cfg.OffsetValue = readCursor(cursorFile)
consumer, err := sarama.NewConsumer(client, topic, partition, "crontab_group", cfg)
if err != nil {
panic(err)
} else {
logPrintln("kafka consumer ready")
}
defer consumer.Close()
consumerLoop:
for {
select {
case <-quit:
logPrintln("kafka consumer quit")
break consumerLoop
case event := <-consumer.Events():
if event.Err != nil {
logPrintln(event.Err)
continue
}
var out map[string]interface{}
err = codec.NewDecoderBytes(event.Value, &codec.MsgpackHandle{}).Decode(&out)
if err != nil {
logPrintln(err)
continue
}
userSceneId := "0"
switch v := out["userSceneId"].(type) {
case uint64:
userSceneId = strconv.FormatInt(int64(v), 10)
case string:
userSceneId = v
case []byte:
userSceneId = string(v)
}
year := time.Now().Year()
timer := string(out["timer"].([]byte))
types := string(out["type"].([]byte))
arr := strings.Split(timer, " ")
if len(arr) == 5 && types == "add" {
if strings.Contains(arr[3], "*") {
timer = strconv.Itoa(rand.Intn(10)) + " " + timer + " *"
} else {
timer = strconv.Itoa(rand.Intn(10)) + " " + timer + " " + strconv.Itoa(year)
}
}
ret := updater(userSceneId+" "+string(out["key"].([]byte)), timer, types, event.Offset)
if ret == true {
updateCursor(event.Offset+1, cursorFile)
}
}
}
}
func (k *KafkaInput) Init(config interface{}) (err error) {
k.config = config.(*KafkaInputConfig)
if len(k.config.Addrs) == 0 {
return errors.New("addrs must have at least one entry")
}
if len(k.config.Group) == 0 {
k.config.Group = k.config.Id
}
k.clientConfig = sarama.NewClientConfig()
k.clientConfig.MetadataRetries = k.config.MetadataRetries
k.clientConfig.WaitForElection = time.Duration(k.config.WaitForElection) * time.Millisecond
k.clientConfig.BackgroundRefreshFrequency = time.Duration(k.config.BackgroundRefreshFrequency) * time.Millisecond
k.clientConfig.DefaultBrokerConf = sarama.NewBrokerConfig()
k.clientConfig.DefaultBrokerConf.MaxOpenRequests = k.config.MaxOpenRequests
k.clientConfig.DefaultBrokerConf.DialTimeout = time.Duration(k.config.DialTimeout) * time.Millisecond
k.clientConfig.DefaultBrokerConf.ReadTimeout = time.Duration(k.config.ReadTimeout) * time.Millisecond
k.clientConfig.DefaultBrokerConf.WriteTimeout = time.Duration(k.config.WriteTimeout) * time.Millisecond
k.consumerConfig = sarama.NewConsumerConfig()
k.consumerConfig.DefaultFetchSize = k.config.DefaultFetchSize
k.consumerConfig.MinFetchSize = k.config.MinFetchSize
k.consumerConfig.MaxMessageSize = k.config.MaxMessageSize
k.consumerConfig.MaxWaitTime = time.Duration(k.config.MaxWaitTime) * time.Millisecond
k.checkpointFilename = k.pConfig.Globals.PrependBaseDir(filepath.Join("kafka",
fmt.Sprintf("%s.%s.%d.offset.bin", k.name, k.config.Topic, k.config.Partition)))
switch k.config.OffsetMethod {
case "Manual":
k.consumerConfig.OffsetMethod = sarama.OffsetMethodManual
if fileExists(k.checkpointFilename) {
if k.consumerConfig.OffsetValue, err = readCheckpoint(k.checkpointFilename); err != nil {
return fmt.Errorf("readCheckpoint %s", err)
}
} else {
if err = os.MkdirAll(filepath.Dir(k.checkpointFilename), 0766); err != nil {
return
}
k.consumerConfig.OffsetMethod = sarama.OffsetMethodOldest
}
case "Newest":
k.consumerConfig.OffsetMethod = sarama.OffsetMethodNewest
if fileExists(k.checkpointFilename) {
if err = os.Remove(k.checkpointFilename); err != nil {
return
}
}
case "Oldest":
k.consumerConfig.OffsetMethod = sarama.OffsetMethodOldest
if fileExists(k.checkpointFilename) {
if err = os.Remove(k.checkpointFilename); err != nil {
return
}
}
default:
return fmt.Errorf("invalid offset_method: %s", k.config.OffsetMethod)
}
k.consumerConfig.EventBufferSize = k.config.EventBufferSize
k.client, err = sarama.NewClient(k.config.Id, k.config.Addrs, k.clientConfig)
if err != nil {
return
}
k.consumer, err = sarama.NewConsumer(k.client, k.config.Topic, k.config.Partition, k.config.Group, k.consumerConfig)
return
}