Пример #1
0
func (this *PartitionController) HandleAppendRequest(request *api.AppendRequest) (*api.AppendReply, error) {

	// TODO: use method that validates the buffer
	messageSet := storage.NewMessageSetFromBuffer(request.Messages)

	this.logger.Withs(tidy.Fields{
		"topic":         request.Topic,
		"partition":     request.Partition,
		"message_count": messageSet.MessageCount(),
	}).Debug("handling append request")

	result := make(chan error, 1)
	this.appendRequests <- &AppendRequest{
		AppendRequest: request,
		MessageSet:    messageSet,
		Result:        result,
	}
	this.logger.Debug("append request send, waiting for result")

	err := <-result

	if err != nil {
		this.logger.WithError(err).Error("append request failed")
	}
	this.logger.WithError(err).Debug("result received")

	this.bytesInRate.Inc(messageSet.DataLen64())
	this.messagesInRate.Inc(int64(messageSet.MessageCount()))

	return &api.AppendReply{
		Ok: err == nil,
	}, nil
}
Пример #2
0
func (this *TopicPartitionConsumer) doReading() {
	//delay := backoff.Exp(time.Millisecond, 1*time.Second)

	//defer close(this.dispatch)
	defer func() {
		close(this.messages)
		logger.Debug("batch consumer done")
	}()

	logger := tidy.GetLogger().Withs(tidy.Fields{
		"host":      this.host,
		"topic":     this.topic,
		"partition": this.partition,
		"offset":    this.offset,
	})
	logger.Debug("reading started")

	replies, err := this.client.Read(context.Background(), &api.ReadRequest{
		Topic:      this.topic,
		Partition:  int32(this.partition),
		Offset:     this.offset.toOffsetData(),
		Continuous: this.continuous,
	})

	if err != nil {
		logger.WithError(err).Error("read request failed")
		return
	}

	for {
		reply, err := replies.Recv()
		if err != nil {
			logger.WithError(err).Error("read request failed")
			return
		}

		if len(reply.Messages) == 0 {
			logger.With("offset", this.offset).Warn("EOF")
			return
		}

		if this.logger.IsDebug() {
			logger.With("offset", this.offset).Debug("reply received")
		}

		this.messages <- IncomingBatch{
			Topic:     this.topic,
			Partition: this.partition,
			Offset:    this.offset,
			Messages:  storage.NewMessageSetFromBuffer(reply.Messages),
		}

		this.offset = offsetFromOffsetData(reply.Offset)
	}
}