示例#1
0
文件: partition.go 项目: pjvds/edgy
func (this *PartitionController) initialize() {
	// TODO: support closing
	this.logger.With("partition", this.ref.String()).Info("initializing partition controller")
	directory := filepath.Join(this.rootDir, this.ref.Topic, this.ref.Partition.String())

	delay := backoff.Exp(1*time.Millisecond, 15*time.Second)

	for {
		storage, err := storage.OpenOrCreatePartition(this.ref, storage.DefaultConfig, directory)

		if err != nil {
			this.logger.WithError(err).Withs(tidy.Fields{
				"partition": this.ref.String(),
				"directory": directory,
				"retry_in":  delay.Next(),
			}).Warn("failed to initialize partition storage")

			delay.Delay()
			continue
		}

		this.storage = storage
		close(this.ready)

		break
	}
}
示例#2
0
文件: controller.go 项目: pjvds/edgy
func (this *Controller) Read(request *api.ReadRequest, stream api.Edgy_ReadServer) error {
	if len(request.Topic) == 0 {
		return errors.New("missing topic")
	}
	if request.Offset == nil {
		return errors.New("missing offset")
	}

	this.logger.Withs(tidy.Fields{
		"topic":      request.Topic,
		"partition":  request.Partition,
		"offset":     tidy.Stringify(request.Offset),
		"continuous": request.Continuous,
	}).Debug("incoming read request")

	ref := storage.PartitionRef{
		Topic:     request.Topic,
		Partition: storage.PartitionId(request.Partition),
	}

	partition, err := this.getPartition(ref)
	if err != nil {
		this.logger.With("partition", ref.String()).WithError(err).Error("failed to get or create storage for partition")
		return err
	}

	this.logger.With("partition", ref).Debug("dispatching request")

	if !request.Continuous {
		_, err := this.executeRead(stream.Context(), partition, request, stream)
		return err
	}

	receiver := partition.Notify(stream.Context())
	errDelay := backoff.Exp(1*time.Millisecond, 5*time.Second)

	for {
		offset, err := this.executeRead(stream.Context(), partition, request, stream)
		if err != nil {
			if err != io.EOF {
				select {
				case <-errDelay.DelayC():
					continue
				case <-stream.Context().Done():
					err := stream.Context().Err()
					this.logger.WithError(err).Warn("unexpected context done signal")
					return err
				}
			}
		} else {
			request.Offset = offset
		}
		errDelay.Reset()

		select {
		case signalOffset, ok := <-receiver.channel:
			this.logger.With("signal_offset", signalOffset).With("ok", ok).Debug("receiver.channel")
			if !ok {
				this.logger.Debug("signal channel closed")
				return nil
			}
			this.logger.With("offset", signalOffset)
			continue
		case <-stream.Context().Done():
			err := stream.Context().Err()
			this.logger.WithError(err).Debug("context done")
			return err
		}
	}
}