func (this *PartitionController) HandleReadRequest(request *api.ReadRequest) (*api.ReadReply, error) {
this.logger.Withs(tidy.Fields{
"topic": request.Topic,
"partition": request.Partition,
"offset": tidy.Stringify(request.Offset),
}).Debug("handling read request")
result, err := this.storage.ReadFrom(storage.Offset{
MessageId: storage.MessageId(request.Offset.MessageId),
SegmentId: storage.SegmentId(request.Offset.SegmentId),
Position: request.Offset.EndPosition,
}, 2*1e6)
if err != nil {
return nil, err
}
this.bytesOutRate.Inc(int64(len(result.Messages)))
this.messagesOutRate.Inc(int64(result.MessageCount))
return &api.ReadReply{
Messages: result.Messages,
Offset: &api.OffsetData{
MessageId: uint64(result.Next.MessageId),
EndPosition: result.Next.Position,
SegmentId: uint64(result.Next.SegmentId),
},
}, nil
}
func NewMessageSetFromBuffer(buffer []byte) *MessageSet {
logger.With("buffer", tidy.Stringify(buffer)).Debug("creating message set from buffer")
position := 0
// TODO: inspect capacity during iteration and grow smarter.
entries := make([]SetEntry, 0, 5)
for position+HEADER_LENGTH < len(buffer) {
if buffer[position+INDEX_START] != START_VALUE {
// TODO: return error
panic(fmt.Errorf("unexpected byte value: expected start value %v at %v, but got %v", START_VALUE, position+INDEX_START, buffer[position+INDEX_START]))
}
entry := SetEntry{
Id: MessageId(byteOrder.Uint64(buffer[position+INDEX_ID:])),
Length: int(byteOrder.Uint32(buffer[position+INDEX_LENGTH:])),
Hash: int64(byteOrder.Uint64(buffer[position+INDEX_HASH:])),
Offset: int(position),
}
valueStart := position + INDEX_CONTENT
valueEnd := valueStart + entry.Length
if valueEnd > len(buffer) {
break
}
entries = append(entries, entry)
position = valueEnd + 1
logger.Withs(tidy.Fields{
"entry": tidy.Stringify(entry),
"position": entry.Offset,
"next_at": position,
}).Debug("entry read")
}
return &MessageSet{
buffer: buffer,
entries: entries,
}
}
func (this *Index) Append(messages *MessageSet) {
for _, entry := range messages.entries {
entry := IndexEntry{
Id: entry.Id,
Offset: entry.Offset, // TODO: this should not equal the set offset
Length: entry.Length,
}
this.items = append(this.items, entry)
logger.With("entry", tidy.Stringify(entry)).Debug("index item appended")
}
}
func (this *Index) FindLastLessOrEqual(id MessageId) (IndexEntry, bool) {
// TODO: add better search algoritm, maybe something as easy as binary search.
for index, entry := range this.items {
if entry.Id < id {
continue
}
if entry.Id == id {
return entry, true
}
if index > 0 {
return this.items[index-1], true
}
}
logger.Withs(tidy.Fields{
"id": id,
"entries": tidy.Stringify(this.items),
}).Debug("no index entry found")
return IndexEntry{}, false
}
// Append writes the messages in the set to the file system. The order is preserved.
func (this *Partition) Append(messages *MessageSet) error {
messages.Align(this.lastMessageId)
segment := this.segments.Last()
if segment == nil {
if rolledTo, err := this.rollToNextSegment(); err != nil {
return err
} else {
segment = rolledTo
}
}
if !segment.SpaceLeftFor(messages) {
this.logger.With("segment", tidy.Stringify(segment)).Debug("no space left for message set in active segment")
if rolledTo, err := this.rollToNextSegment(); err != nil {
return err
} else {
segment = rolledTo
}
}
this.logger.With("segment", segment.ref).Debug("writting to segment")
if err := segment.Append(messages); err != nil {
this.logger.With("segment", segment.ref).WithError(err).Warn("write to segment failed")
return err
}
this.logger.With("segment", segment.ref).Debug("write to segment success")
this.lastMessageId = this.lastMessageId.NextN(messages.MessageCount())
this.logger.With("last_message_id", this.lastMessageId).Info("append success")
return nil
}
func (this *Partition) ReadFrom(offset Offset, eagerFetchUntilMaxBytes int) (ReadResult, error) {
this.logger.Withs(tidy.Fields{
"offset": tidy.Stringify(offset),
}).Debug("handling ReadFrom")
if offset.IsEmpty() {
offset.MessageId = MessageId(1)
offset.SegmentId = SegmentId(1)
offset.Position = 0
}
if offset.IsEndOfStream() {
return ReadResult{
Next: this.getLastOffset(),
}, nil
}
if this.closed {
this.logger.WithError(ErrClosed).Debug("ReadFrom called while closed")
return ReadResult{Next: offset}, ErrClosed
}
// TODO: return empty set when offset is beyond lastMessageId of the writer.
buffer := make([]byte, eagerFetchUntilMaxBytes)
for {
segment, ok := this.segments.Get(offset.SegmentId)
if !ok {
err := errors.New("segment not found")
this.logger.Withs(tidy.Fields{
"offset": offset,
}).WithError(err).Error("ReadFrom failed")
return ReadResult{}, err
}
read, err := segment.ReadAt(buffer, offset.Position)
if err != nil && err != io.EOF {
logger.WithError(err).Withs(tidy.Fields{
"segment": segment.ref,
"buffer_length": len(buffer),
"bytes_read": read,
}).Error("failed to read from segment file")
return ReadResult{}, err
}
if err == io.EOF && read == 0 {
logger.WithError(err).Withs(tidy.Fields{
"segment": segment.ref,
"buffer_length": len(buffer),
"bytes_read": read,
"position": offset.Position,
}).Error("unexpected EOF")
return ReadResult{}, err
}
if buffer[0] == END_OF_SEGMENT {
offset = Offset{
MessageId: offset.MessageId,
SegmentId: SegmentId(offset.MessageId),
Position: 0,
}
continue
}
if buffer[0] != START_VALUE {
if buffer[0] == 0x00 {
return ReadResult{
Messages: make([]byte, 0),
Next: offset,
}, nil
}
err := errors.New("unexpected start value")
logger.WithError(err).Withs(tidy.Fields{
"segment": segment.ref,
"offset": offset,
"first_byte": buffer[0],
}).Error("read failure")
return ReadResult{}, err
}
// we are having something in the buffer
// if buffer[0] != START_VALUE {
// this.logger.Withs(tidy.Fields{
// "segment_id": segmentId,
// "position": position,
// "offset": tidy.Stringify(offset),
// "byte_value": buffer[0],
// }).Error("no message found at current position")
//
// return result, errors.New("no message found at current position")
// }
position := 0
messageCount := 0
for {
if position > read-HEADER_LENGTH {
break
}
header := ReadHeader(buffer[position:])
if header.Magic != START_VALUE {
// we reached a point where there is no message header
// if header.Magic == END_OF_SEGMENT {
// result.Next.SegmentId = SegmentId(result.Next.MessageId)
// result.Next.Position = 0
// }
break
}
if header.MessageId != offset.MessageId {
this.logger.Withs(tidy.Fields{
"offset": offset,
"expected_message_id": offset.MessageId,
"actual_message_id": header.MessageId,
"position": position,
"segment": segment.ref,
}).Error("unexpected message id")
return ReadResult{}, errors.New("data error")
}
if header.ContentLength > int32(read-position-HEADER_LENGTH) {
// message content is not, or partial in buffer
break
}
offset.MessageId = header.MessageId.Next()
offset.Position += int64(HEADER_LENGTH + header.ContentLength)
position += int(HEADER_LENGTH + header.ContentLength)
messageCount++
}
return ReadResult{
MessageCount: messageCount,
Messages: buffer[0:position],
Next: offset,
}, nil
}
}
func (this *Controller) Read(request *api.ReadRequest, stream api.Edgy_ReadServer) error {
if len(request.Topic) == 0 {
return errors.New("missing topic")
}
if request.Offset == nil {
return errors.New("missing offset")
}
this.logger.Withs(tidy.Fields{
"topic": request.Topic,
"partition": request.Partition,
"offset": tidy.Stringify(request.Offset),
"continuous": request.Continuous,
}).Debug("incoming read request")
ref := storage.PartitionRef{
Topic: request.Topic,
Partition: storage.PartitionId(request.Partition),
}
partition, err := this.getPartition(ref)
if err != nil {
this.logger.With("partition", ref.String()).WithError(err).Error("failed to get or create storage for partition")
return err
}
this.logger.With("partition", ref).Debug("dispatching request")
if !request.Continuous {
_, err := this.executeRead(stream.Context(), partition, request, stream)
return err
}
receiver := partition.Notify(stream.Context())
errDelay := backoff.Exp(1*time.Millisecond, 5*time.Second)
for {
offset, err := this.executeRead(stream.Context(), partition, request, stream)
if err != nil {
if err != io.EOF {
select {
case <-errDelay.DelayC():
continue
case <-stream.Context().Done():
err := stream.Context().Err()
this.logger.WithError(err).Warn("unexpected context done signal")
return err
}
}
} else {
request.Offset = offset
}
errDelay.Reset()
select {
case signalOffset, ok := <-receiver.channel:
this.logger.With("signal_offset", signalOffset).With("ok", ok).Debug("receiver.channel")
if !ok {
this.logger.Debug("signal channel closed")
return nil
}
this.logger.With("offset", signalOffset)
continue
case <-stream.Context().Done():
err := stream.Context().Err()
this.logger.WithError(err).Debug("context done")
return err
}
}
}