Example #1
0
func (this *PartitionController) HandleReadRequest(request *api.ReadRequest) (*api.ReadReply, error) {
	this.logger.Withs(tidy.Fields{
		"topic":     request.Topic,
		"partition": request.Partition,
		"offset":    tidy.Stringify(request.Offset),
	}).Debug("handling read request")

	result, err := this.storage.ReadFrom(storage.Offset{
		MessageId: storage.MessageId(request.Offset.MessageId),
		SegmentId: storage.SegmentId(request.Offset.SegmentId),
		Position:  request.Offset.EndPosition,
	}, 2*1e6)
	if err != nil {
		return nil, err
	}

	this.bytesOutRate.Inc(int64(len(result.Messages)))
	this.messagesOutRate.Inc(int64(result.MessageCount))

	return &api.ReadReply{
		Messages: result.Messages,
		Offset: &api.OffsetData{
			MessageId:   uint64(result.Next.MessageId),
			EndPosition: result.Next.Position,
			SegmentId:   uint64(result.Next.SegmentId),
		},
	}, nil
}
Example #2
0
func NewMessageSetFromBuffer(buffer []byte) *MessageSet {
	logger.With("buffer", tidy.Stringify(buffer)).Debug("creating message set from buffer")
	position := 0

	// TODO: inspect capacity during iteration and grow smarter.
	entries := make([]SetEntry, 0, 5)

	for position+HEADER_LENGTH < len(buffer) {
		if buffer[position+INDEX_START] != START_VALUE {
			// TODO: return error
			panic(fmt.Errorf("unexpected byte value: expected start value %v at %v, but got %v", START_VALUE, position+INDEX_START, buffer[position+INDEX_START]))
		}

		entry := SetEntry{
			Id:     MessageId(byteOrder.Uint64(buffer[position+INDEX_ID:])),
			Length: int(byteOrder.Uint32(buffer[position+INDEX_LENGTH:])),
			Hash:   int64(byteOrder.Uint64(buffer[position+INDEX_HASH:])),
			Offset: int(position),
		}

		valueStart := position + INDEX_CONTENT
		valueEnd := valueStart + entry.Length

		if valueEnd > len(buffer) {
			break
		}

		entries = append(entries, entry)

		position = valueEnd + 1

		logger.Withs(tidy.Fields{
			"entry":    tidy.Stringify(entry),
			"position": entry.Offset,
			"next_at":  position,
		}).Debug("entry read")
	}

	return &MessageSet{
		buffer:  buffer,
		entries: entries,
	}
}
Example #3
0
func (this *Index) Append(messages *MessageSet) {
	for _, entry := range messages.entries {
		entry := IndexEntry{
			Id:     entry.Id,
			Offset: entry.Offset, // TODO: this should not equal the set offset
			Length: entry.Length,
		}

		this.items = append(this.items, entry)
		logger.With("entry", tidy.Stringify(entry)).Debug("index item appended")
	}
}
Example #4
0
func (this *Index) FindLastLessOrEqual(id MessageId) (IndexEntry, bool) {
	// TODO: add better search algoritm, maybe something as easy as binary search.
	for index, entry := range this.items {
		if entry.Id < id {
			continue
		}
		if entry.Id == id {
			return entry, true
		}
		if index > 0 {
			return this.items[index-1], true
		}
	}

	logger.Withs(tidy.Fields{
		"id":      id,
		"entries": tidy.Stringify(this.items),
	}).Debug("no index entry found")

	return IndexEntry{}, false
}
Example #5
0
// Append writes the messages in the set to the file system. The order is preserved.
func (this *Partition) Append(messages *MessageSet) error {
	messages.Align(this.lastMessageId)
	segment := this.segments.Last()

	if segment == nil {
		if rolledTo, err := this.rollToNextSegment(); err != nil {
			return err
		} else {
			segment = rolledTo
		}
	}

	if !segment.SpaceLeftFor(messages) {
		this.logger.With("segment", tidy.Stringify(segment)).Debug("no space left for message set in active segment")

		if rolledTo, err := this.rollToNextSegment(); err != nil {
			return err
		} else {
			segment = rolledTo
		}
	}

	this.logger.With("segment", segment.ref).Debug("writting to segment")

	if err := segment.Append(messages); err != nil {
		this.logger.With("segment", segment.ref).WithError(err).Warn("write to segment failed")

		return err
	}
	this.logger.With("segment", segment.ref).Debug("write to segment success")

	this.lastMessageId = this.lastMessageId.NextN(messages.MessageCount())
	this.logger.With("last_message_id", this.lastMessageId).Info("append success")

	return nil
}
Example #6
0
func (this *Partition) ReadFrom(offset Offset, eagerFetchUntilMaxBytes int) (ReadResult, error) {
	this.logger.Withs(tidy.Fields{
		"offset": tidy.Stringify(offset),
	}).Debug("handling ReadFrom")

	if offset.IsEmpty() {
		offset.MessageId = MessageId(1)
		offset.SegmentId = SegmentId(1)
		offset.Position = 0
	}

	if offset.IsEndOfStream() {
		return ReadResult{
			Next: this.getLastOffset(),
		}, nil
	}

	if this.closed {
		this.logger.WithError(ErrClosed).Debug("ReadFrom called while closed")
		return ReadResult{Next: offset}, ErrClosed
	}
	// TODO: return empty set when offset is beyond lastMessageId of the writer.

	buffer := make([]byte, eagerFetchUntilMaxBytes)

	for {
		segment, ok := this.segments.Get(offset.SegmentId)
		if !ok {
			err := errors.New("segment not found")

			this.logger.Withs(tidy.Fields{
				"offset": offset,
			}).WithError(err).Error("ReadFrom failed")

			return ReadResult{}, err
		}

		read, err := segment.ReadAt(buffer, offset.Position)

		if err != nil && err != io.EOF {
			logger.WithError(err).Withs(tidy.Fields{
				"segment":       segment.ref,
				"buffer_length": len(buffer),
				"bytes_read":    read,
			}).Error("failed to read from segment file")

			return ReadResult{}, err
		}

		if err == io.EOF && read == 0 {
			logger.WithError(err).Withs(tidy.Fields{
				"segment":       segment.ref,
				"buffer_length": len(buffer),
				"bytes_read":    read,
				"position":      offset.Position,
			}).Error("unexpected EOF")

			return ReadResult{}, err
		}

		if buffer[0] == END_OF_SEGMENT {
			offset = Offset{
				MessageId: offset.MessageId,
				SegmentId: SegmentId(offset.MessageId),
				Position:  0,
			}
			continue
		}

		if buffer[0] != START_VALUE {
			if buffer[0] == 0x00 {
				return ReadResult{
					Messages: make([]byte, 0),
					Next:     offset,
				}, nil
			}

			err := errors.New("unexpected start value")

			logger.WithError(err).Withs(tidy.Fields{
				"segment":    segment.ref,
				"offset":     offset,
				"first_byte": buffer[0],
			}).Error("read failure")

			return ReadResult{}, err
		}

		// we are having something in the buffer
		// if buffer[0] != START_VALUE {
		// 	this.logger.Withs(tidy.Fields{
		// 		"segment_id": segmentId,
		// 		"position":   position,
		// 		"offset":     tidy.Stringify(offset),
		// 		"byte_value": buffer[0],
		// 	}).Error("no message found at current position")
		//
		// 	return result, errors.New("no message found at current position")
		// }

		position := 0
		messageCount := 0

		for {
			if position > read-HEADER_LENGTH {
				break
			}

			header := ReadHeader(buffer[position:])

			if header.Magic != START_VALUE {
				// we reached a point where there is no message header
				// if header.Magic == END_OF_SEGMENT {
				// 	result.Next.SegmentId = SegmentId(result.Next.MessageId)
				// 	result.Next.Position = 0
				// }
				break
			}

			if header.MessageId != offset.MessageId {
				this.logger.Withs(tidy.Fields{
					"offset":              offset,
					"expected_message_id": offset.MessageId,
					"actual_message_id":   header.MessageId,
					"position":            position,
					"segment":             segment.ref,
				}).Error("unexpected message id")

				return ReadResult{}, errors.New("data error")
			}

			if header.ContentLength > int32(read-position-HEADER_LENGTH) {
				// message content is not, or partial in buffer
				break
			}

			offset.MessageId = header.MessageId.Next()
			offset.Position += int64(HEADER_LENGTH + header.ContentLength)

			position += int(HEADER_LENGTH + header.ContentLength)
			messageCount++
		}

		return ReadResult{
			MessageCount: messageCount,
			Messages:     buffer[0:position],
			Next:         offset,
		}, nil
	}
}
Example #7
0
func (this *Controller) Read(request *api.ReadRequest, stream api.Edgy_ReadServer) error {
	if len(request.Topic) == 0 {
		return errors.New("missing topic")
	}
	if request.Offset == nil {
		return errors.New("missing offset")
	}

	this.logger.Withs(tidy.Fields{
		"topic":      request.Topic,
		"partition":  request.Partition,
		"offset":     tidy.Stringify(request.Offset),
		"continuous": request.Continuous,
	}).Debug("incoming read request")

	ref := storage.PartitionRef{
		Topic:     request.Topic,
		Partition: storage.PartitionId(request.Partition),
	}

	partition, err := this.getPartition(ref)
	if err != nil {
		this.logger.With("partition", ref.String()).WithError(err).Error("failed to get or create storage for partition")
		return err
	}

	this.logger.With("partition", ref).Debug("dispatching request")

	if !request.Continuous {
		_, err := this.executeRead(stream.Context(), partition, request, stream)
		return err
	}

	receiver := partition.Notify(stream.Context())
	errDelay := backoff.Exp(1*time.Millisecond, 5*time.Second)

	for {
		offset, err := this.executeRead(stream.Context(), partition, request, stream)
		if err != nil {
			if err != io.EOF {
				select {
				case <-errDelay.DelayC():
					continue
				case <-stream.Context().Done():
					err := stream.Context().Err()
					this.logger.WithError(err).Warn("unexpected context done signal")
					return err
				}
			}
		} else {
			request.Offset = offset
		}
		errDelay.Reset()

		select {
		case signalOffset, ok := <-receiver.channel:
			this.logger.With("signal_offset", signalOffset).With("ok", ok).Debug("receiver.channel")
			if !ok {
				this.logger.Debug("signal channel closed")
				return nil
			}
			this.logger.With("offset", signalOffset)
			continue
		case <-stream.Context().Done():
			err := stream.Context().Err()
			this.logger.WithError(err).Debug("context done")
			return err
		}
	}
}