// LogError - write error to zap log func LogError(logger zap.Logger, err error) { werr, ok := err.(*ErrorEx) if !ok { logger.Error(err.Error()) } else { logger.Log(werr.Level, werr.Error(), werr.Fields...) } }
func (cg *ConsumerGroup) reload(logger zap.Logger) error { cg.reloadMutex.Lock() defer cg.reloadMutex.Unlock() cg.singleReload.Do(func() { logger.Info("KAFKA: Closing down old connections for replica", zap.Int("replicaId", cg.replicaId), ) err := cg.Close(logger) if err != nil { logger.Error("KAFKA: Failed to close consumergroup for replica", zap.Int("replicaId", cg.replicaId), zap.Error(err), ) } cg.Load(logger) }) return nil }
func (cg *ConsumerGroup) GetBatchOfMessages(batchSize int, logger zap.Logger) []string { offsets := make(map[string]map[int32]int64) groupOfMessages := []string{} if batchSize == 0 { return groupOfMessages } counter := 0 for { if counter == batchSize { break } cg.reloadMutex.Lock() select { case message := <-cg.messages: if offsets[message.Topic] == nil { offsets[message.Topic] = make(map[int32]int64) } if offsets[message.Topic][message.Partition] != 0 && offsets[message.Topic][message.Partition] != message.Offset-1 { logger.Error("KAFKA: Unexpected offset for message topic and partition", zap.String("messageTopic", message.Topic), zap.Int64("messagePartition", int64(message.Partition)), zap.Int64("offsetExpected", offsets[message.Topic][message.Partition]+1), zap.Int64("offsetFound", message.Offset), zap.Int64("offsetDifference", message.Offset-offsets[message.Topic][message.Partition]+1), ) continue } groupOfMessages = append(groupOfMessages, string(message.Value)) offsets[message.Topic][message.Partition] = message.Offset cg.CommitUpto(message) counter += 1 cg.reloadMutex.Unlock() default: cg.reloadMutex.Unlock() continue } } return groupOfMessages }
func (cg *ConsumerGroup) Close(logger zap.Logger) error { shutdownError := AlreadyClosing cg.singleShutdown.Do(func() { defer cg.kazoo.Close() shutdownError = nil close(cg.stopper) cg.wg.Wait() if err := cg.offsetManager.Close(logger); err != nil { logger.Error("KAFKA: FAILED closing the offset manager for replica!", zap.Int("replicaId", cg.replicaId), zap.Error(err), ) } if shutdownError = cg.instance.Deregister(); shutdownError != nil { logger.Warn("KAFKA: Replica FAILED deregistering consumer instance", zap.Int("replicaId", cg.replicaId), zap.Error(shutdownError), ) } else { logger.Info("KAFKA: Replica deregistered consumer instance", zap.Int("replicaId", cg.replicaId), zap.String("instanceId", cg.instance.ID), ) } if shutdownError = cg.consumer.Close(); shutdownError != nil { logger.Error("Replica FAILED closing the Sarama client", zap.Int("replicaId", cg.replicaId), zap.Error(shutdownError), ) } close(cg.messages) close(cg.errors) cg.instance = nil }) return shutdownError }
// Consumes a partition func (cg *ConsumerGroup) partitionConsumer(topic string, partition int32, messages chan<- *sarama.ConsumerMessage, errors chan<- *sarama.ConsumerError, wg *sync.WaitGroup, stopper <-chan struct{}, logger zap.Logger) { defer wg.Done() select { case <-stopper: return default: } for maxRetries, tries := 3, 0; tries < maxRetries; tries++ { if err := cg.instance.ClaimPartition(topic, partition); err == nil { break } else if err == kazoo.ErrPartitionClaimedByOther && tries+1 < maxRetries { time.Sleep(1 * time.Second) } else { logger.Warn("KAFKA: Replica FAILED to claim partition", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), zap.Error(err), ) return } } defer cg.instance.ReleasePartition(topic, partition) nextOffset, err := cg.offsetManager.InitializePartition(topic, partition) if err != nil { logger.Error("KAFKA: Replica FAILED to determine initial offset", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), zap.Error(err), ) return } if nextOffset >= 0 { logger.Info("KAFKA: Replica partition consumer starting at offset", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), zap.Int64("nextOffset", nextOffset), ) } else { nextOffset = cg.config.Offsets.Initial if nextOffset == sarama.OffsetOldest { logger.Info("KAFKA: Replica partition consumer starting at the oldest available offset", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), ) } else if nextOffset == sarama.OffsetNewest { logger.Info("KAFKA: Replica partition consumer listening for new messages only", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), ) } } consumer, err := cg.consumer.ConsumePartition(topic, partition, nextOffset) if err == sarama.ErrOffsetOutOfRange { logger.Warn("KAFKA: Replica partition consumer offset out of Range", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), ) // if the offset is out of range, simplistically decide whether to use OffsetNewest or OffsetOldest // if the configuration specified offsetOldest, then switch to the oldest available offset, else // switch to the newest available offset. if cg.config.Offsets.Initial == sarama.OffsetOldest { nextOffset = sarama.OffsetOldest logger.Info("KAFKA: Replica partition consumer offset reset to oldest available offset", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), ) } else { nextOffset = sarama.OffsetNewest logger.Info("KAFKA: Replica partition consumer offset reset to newest available offset", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), ) } // retry the consumePartition with the adjusted offset consumer, err = cg.consumer.ConsumePartition(topic, partition, nextOffset) } if err != nil { logger.Fatal("KAFKA: Replica FAILED to start partition consumer", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), zap.Error(err), ) return } defer consumer.Close() err = nil var lastOffset int64 = -1 // aka unknown partitionConsumerLoop: for { select { case <-stopper: break partitionConsumerLoop case err := <-consumer.Errors(): for { select { case errors <- err: continue partitionConsumerLoop case <-stopper: break partitionConsumerLoop } } case message := <-consumer.Messages(): for { select { case <-stopper: break partitionConsumerLoop case messages <- message: lastOffset = message.Offset continue partitionConsumerLoop } } } } logger.Info("KAFKA: Replica is stopping partition consumer at offset", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), zap.Int64("lastOffset", lastOffset), ) if err = cg.offsetManager.FinalizePartition(topic, partition, lastOffset, cg.config.Offsets.ProcessingTimeout, cg.replicaId, logger); err != nil { logger.Fatal("KAFKA: Replica error trying to stop partition consumer", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), zap.Error(err), ) } logger.Info("KAFKA: Replica successfully stoped partition", zap.Int("replicaId", cg.replicaId), zap.String("topic", topic), zap.Int64("partition", int64(partition)), ) }