// 172.31.22.222:2181,172.31.21.206:2181,172.31.17.130:2181
func NewConsumer(group string, topics, zk []string, strategy kafkalib.WorkerStrategy) (*Consumer, error) {
consumer := &Consumer{
topics: topics,
}
coordinatorConfig := kafkalib.NewZookeeperConfig()
coordinatorConfig.ZookeeperConnect = zk
coordinatorConfig.Root = "/kafka"
coordinator := kafkalib.NewZookeeperCoordinator(coordinatorConfig)
consumerConfig := kafkalib.DefaultConsumerConfig()
consumerConfig.Groupid = group
consumerConfig.Coordinator = coordinator
consumerConfig.Strategy = strategy
consumerConfig.AutoOffsetReset = kafkalib.SmallestOffset
consumerConfig.OffsetCommitInterval = 1 * time.Minute
consumerConfig.OffsetsCommitMaxRetries = 5
consumerConfig.MaxWorkerRetries = 5
consumerConfig.WorkerFailureCallback = func(wm *kafkalib.WorkerManager) kafkalib.FailedDecision {
kafkalib.Error(consumer, "Failed to write . Shutting down...")
return kafkalib.DoNotCommitOffsetAndStop
}
consumerConfig.WorkerFailedAttemptCallback = func(task *kafkalib.Task, result kafkalib.WorkerResult) kafkalib.FailedDecision {
kafkalib.Errorf(consumer, "Failed to write %s to the database after %d retries", task.Id().String(), task.Retries)
return kafkalib.DoNotCommitOffsetAndContinue
}
consumer.config = consumerConfig
consumer.consumer = kafkalib.NewConsumer(consumerConfig)
return consumer, nil
}
func protobufTransformer(msg *kafka.SyslogMessage, topic string) *sarama.ProducerMessage {
line := &sp.LogLine{}
line.Line = proto.String(msg.Message)
line.Source = proto.String(*source)
for k, v := range tag {
line.Tag = append(line.Tag, &sp.LogLine_Tag{Key: proto.String(k), Value: proto.String(v)})
}
if *logtypeid != math.MinInt64 {
line.Logtypeid = logtypeid
}
line.Timings = append(line.Timings, msg.Timestamp, time.Now().UnixNano()/int64(time.Millisecond))
protobuf, err := proto.Marshal(line)
if err != nil {
kafka.Errorf("protobuf-transformer", "Failed to marshal %s as Protocol Buffer", msg)
}
return &sarama.ProducerMessage{Topic: topic, Key: sarama.StringEncoder(*source), Value: sarama.ByteEncoder(protobuf)}
}
