func (this *PartitionController) HandleReadRequest(request *api.ReadRequest) (*api.ReadReply, error) { this.logger.Withs(tidy.Fields{ "topic": request.Topic, "partition": request.Partition, "offset": tidy.Stringify(request.Offset), }).Debug("handling read request") result, err := this.storage.ReadFrom(storage.Offset{ MessageId: storage.MessageId(request.Offset.MessageId), SegmentId: storage.SegmentId(request.Offset.SegmentId), Position: request.Offset.EndPosition, }, 2*1e6) if err != nil { return nil, err } this.bytesOutRate.Inc(int64(len(result.Messages))) this.messagesOutRate.Inc(int64(result.MessageCount)) return &api.ReadReply{ Messages: result.Messages, Offset: &api.OffsetData{ MessageId: uint64(result.Next.MessageId), EndPosition: result.Next.Position, SegmentId: uint64(result.Next.SegmentId), }, }, nil }
func NewMessageSetFromBuffer(buffer []byte) *MessageSet { logger.With("buffer", tidy.Stringify(buffer)).Debug("creating message set from buffer") position := 0 // TODO: inspect capacity during iteration and grow smarter. entries := make([]SetEntry, 0, 5) for position+HEADER_LENGTH < len(buffer) { if buffer[position+INDEX_START] != START_VALUE { // TODO: return error panic(fmt.Errorf("unexpected byte value: expected start value %v at %v, but got %v", START_VALUE, position+INDEX_START, buffer[position+INDEX_START])) } entry := SetEntry{ Id: MessageId(byteOrder.Uint64(buffer[position+INDEX_ID:])), Length: int(byteOrder.Uint32(buffer[position+INDEX_LENGTH:])), Hash: int64(byteOrder.Uint64(buffer[position+INDEX_HASH:])), Offset: int(position), } valueStart := position + INDEX_CONTENT valueEnd := valueStart + entry.Length if valueEnd > len(buffer) { break } entries = append(entries, entry) position = valueEnd + 1 logger.Withs(tidy.Fields{ "entry": tidy.Stringify(entry), "position": entry.Offset, "next_at": position, }).Debug("entry read") } return &MessageSet{ buffer: buffer, entries: entries, } }
func (this *Index) Append(messages *MessageSet) { for _, entry := range messages.entries { entry := IndexEntry{ Id: entry.Id, Offset: entry.Offset, // TODO: this should not equal the set offset Length: entry.Length, } this.items = append(this.items, entry) logger.With("entry", tidy.Stringify(entry)).Debug("index item appended") } }
func (this *Index) FindLastLessOrEqual(id MessageId) (IndexEntry, bool) { // TODO: add better search algoritm, maybe something as easy as binary search. for index, entry := range this.items { if entry.Id < id { continue } if entry.Id == id { return entry, true } if index > 0 { return this.items[index-1], true } } logger.Withs(tidy.Fields{ "id": id, "entries": tidy.Stringify(this.items), }).Debug("no index entry found") return IndexEntry{}, false }
// Append writes the messages in the set to the file system. The order is preserved. func (this *Partition) Append(messages *MessageSet) error { messages.Align(this.lastMessageId) segment := this.segments.Last() if segment == nil { if rolledTo, err := this.rollToNextSegment(); err != nil { return err } else { segment = rolledTo } } if !segment.SpaceLeftFor(messages) { this.logger.With("segment", tidy.Stringify(segment)).Debug("no space left for message set in active segment") if rolledTo, err := this.rollToNextSegment(); err != nil { return err } else { segment = rolledTo } } this.logger.With("segment", segment.ref).Debug("writting to segment") if err := segment.Append(messages); err != nil { this.logger.With("segment", segment.ref).WithError(err).Warn("write to segment failed") return err } this.logger.With("segment", segment.ref).Debug("write to segment success") this.lastMessageId = this.lastMessageId.NextN(messages.MessageCount()) this.logger.With("last_message_id", this.lastMessageId).Info("append success") return nil }
func (this *Partition) ReadFrom(offset Offset, eagerFetchUntilMaxBytes int) (ReadResult, error) { this.logger.Withs(tidy.Fields{ "offset": tidy.Stringify(offset), }).Debug("handling ReadFrom") if offset.IsEmpty() { offset.MessageId = MessageId(1) offset.SegmentId = SegmentId(1) offset.Position = 0 } if offset.IsEndOfStream() { return ReadResult{ Next: this.getLastOffset(), }, nil } if this.closed { this.logger.WithError(ErrClosed).Debug("ReadFrom called while closed") return ReadResult{Next: offset}, ErrClosed } // TODO: return empty set when offset is beyond lastMessageId of the writer. buffer := make([]byte, eagerFetchUntilMaxBytes) for { segment, ok := this.segments.Get(offset.SegmentId) if !ok { err := errors.New("segment not found") this.logger.Withs(tidy.Fields{ "offset": offset, }).WithError(err).Error("ReadFrom failed") return ReadResult{}, err } read, err := segment.ReadAt(buffer, offset.Position) if err != nil && err != io.EOF { logger.WithError(err).Withs(tidy.Fields{ "segment": segment.ref, "buffer_length": len(buffer), "bytes_read": read, }).Error("failed to read from segment file") return ReadResult{}, err } if err == io.EOF && read == 0 { logger.WithError(err).Withs(tidy.Fields{ "segment": segment.ref, "buffer_length": len(buffer), "bytes_read": read, "position": offset.Position, }).Error("unexpected EOF") return ReadResult{}, err } if buffer[0] == END_OF_SEGMENT { offset = Offset{ MessageId: offset.MessageId, SegmentId: SegmentId(offset.MessageId), Position: 0, } continue } if buffer[0] != START_VALUE { if buffer[0] == 0x00 { return ReadResult{ Messages: make([]byte, 0), Next: offset, }, nil } err := errors.New("unexpected start value") logger.WithError(err).Withs(tidy.Fields{ "segment": segment.ref, "offset": offset, "first_byte": buffer[0], }).Error("read failure") return ReadResult{}, err } // we are having something in the buffer // if buffer[0] != START_VALUE { // this.logger.Withs(tidy.Fields{ // "segment_id": segmentId, // "position": position, // "offset": tidy.Stringify(offset), // "byte_value": buffer[0], // }).Error("no message found at current position") // // return result, errors.New("no message found at current position") // } position := 0 messageCount := 0 for { if position > read-HEADER_LENGTH { break } header := ReadHeader(buffer[position:]) if header.Magic != START_VALUE { // we reached a point where there is no message header // if header.Magic == END_OF_SEGMENT { // result.Next.SegmentId = SegmentId(result.Next.MessageId) // result.Next.Position = 0 // } break } if header.MessageId != offset.MessageId { this.logger.Withs(tidy.Fields{ "offset": offset, "expected_message_id": offset.MessageId, "actual_message_id": header.MessageId, "position": position, "segment": segment.ref, }).Error("unexpected message id") return ReadResult{}, errors.New("data error") } if header.ContentLength > int32(read-position-HEADER_LENGTH) { // message content is not, or partial in buffer break } offset.MessageId = header.MessageId.Next() offset.Position += int64(HEADER_LENGTH + header.ContentLength) position += int(HEADER_LENGTH + header.ContentLength) messageCount++ } return ReadResult{ MessageCount: messageCount, Messages: buffer[0:position], Next: offset, }, nil } }
func (this *Controller) Read(request *api.ReadRequest, stream api.Edgy_ReadServer) error { if len(request.Topic) == 0 { return errors.New("missing topic") } if request.Offset == nil { return errors.New("missing offset") } this.logger.Withs(tidy.Fields{ "topic": request.Topic, "partition": request.Partition, "offset": tidy.Stringify(request.Offset), "continuous": request.Continuous, }).Debug("incoming read request") ref := storage.PartitionRef{ Topic: request.Topic, Partition: storage.PartitionId(request.Partition), } partition, err := this.getPartition(ref) if err != nil { this.logger.With("partition", ref.String()).WithError(err).Error("failed to get or create storage for partition") return err } this.logger.With("partition", ref).Debug("dispatching request") if !request.Continuous { _, err := this.executeRead(stream.Context(), partition, request, stream) return err } receiver := partition.Notify(stream.Context()) errDelay := backoff.Exp(1*time.Millisecond, 5*time.Second) for { offset, err := this.executeRead(stream.Context(), partition, request, stream) if err != nil { if err != io.EOF { select { case <-errDelay.DelayC(): continue case <-stream.Context().Done(): err := stream.Context().Err() this.logger.WithError(err).Warn("unexpected context done signal") return err } } } else { request.Offset = offset } errDelay.Reset() select { case signalOffset, ok := <-receiver.channel: this.logger.With("signal_offset", signalOffset).With("ok", ok).Debug("receiver.channel") if !ok { this.logger.Debug("signal channel closed") return nil } this.logger.With("offset", signalOffset) continue case <-stream.Context().Done(): err := stream.Context().Err() this.logger.WithError(err).Debug("context done") return err } } }