func (this *PartitionController) HandleAppendRequest(request *api.AppendRequest) (*api.AppendReply, error) {
// TODO: use method that validates the buffer
messageSet := storage.NewMessageSetFromBuffer(request.Messages)
this.logger.Withs(tidy.Fields{
"topic": request.Topic,
"partition": request.Partition,
"message_count": messageSet.MessageCount(),
}).Debug("handling append request")
result := make(chan error, 1)
this.appendRequests <- &AppendRequest{
AppendRequest: request,
MessageSet: messageSet,
Result: result,
}
this.logger.Debug("append request send, waiting for result")
err := <-result
if err != nil {
this.logger.WithError(err).Error("append request failed")
}
this.logger.WithError(err).Debug("result received")
this.bytesInRate.Inc(messageSet.DataLen64())
this.messagesInRate.Inc(int64(messageSet.MessageCount()))
return &api.AppendReply{
Ok: err == nil,
}, nil
}
func (this *TopicPartitionConsumer) doReading() {
//delay := backoff.Exp(time.Millisecond, 1*time.Second)
//defer close(this.dispatch)
defer func() {
close(this.messages)
logger.Debug("batch consumer done")
}()
logger := tidy.GetLogger().Withs(tidy.Fields{
"host": this.host,
"topic": this.topic,
"partition": this.partition,
"offset": this.offset,
})
logger.Debug("reading started")
replies, err := this.client.Read(context.Background(), &api.ReadRequest{
Topic: this.topic,
Partition: int32(this.partition),
Offset: this.offset.toOffsetData(),
Continuous: this.continuous,
})
if err != nil {
logger.WithError(err).Error("read request failed")
return
}
for {
reply, err := replies.Recv()
if err != nil {
logger.WithError(err).Error("read request failed")
return
}
if len(reply.Messages) == 0 {
logger.With("offset", this.offset).Warn("EOF")
return
}
if this.logger.IsDebug() {
logger.With("offset", this.offset).Debug("reply received")
}
this.messages <- IncomingBatch{
Topic: this.topic,
Partition: this.partition,
Offset: this.offset,
Messages: storage.NewMessageSetFromBuffer(reply.Messages),
}
this.offset = offsetFromOffsetData(reply.Offset)
}
}