func (this *PartitionController) initialize() {
// TODO: support closing
this.logger.With("partition", this.ref.String()).Info("initializing partition controller")
directory := filepath.Join(this.rootDir, this.ref.Topic, this.ref.Partition.String())
delay := backoff.Exp(1*time.Millisecond, 15*time.Second)
for {
storage, err := storage.OpenOrCreatePartition(this.ref, storage.DefaultConfig, directory)
if err != nil {
this.logger.WithError(err).Withs(tidy.Fields{
"partition": this.ref.String(),
"directory": directory,
"retry_in": delay.Next(),
}).Warn("failed to initialize partition storage")
delay.Delay()
continue
}
this.storage = storage
close(this.ready)
break
}
}
func (this *Controller) Read(request *api.ReadRequest, stream api.Edgy_ReadServer) error {
if len(request.Topic) == 0 {
return errors.New("missing topic")
}
if request.Offset == nil {
return errors.New("missing offset")
}
this.logger.Withs(tidy.Fields{
"topic": request.Topic,
"partition": request.Partition,
"offset": tidy.Stringify(request.Offset),
"continuous": request.Continuous,
}).Debug("incoming read request")
ref := storage.PartitionRef{
Topic: request.Topic,
Partition: storage.PartitionId(request.Partition),
}
partition, err := this.getPartition(ref)
if err != nil {
this.logger.With("partition", ref.String()).WithError(err).Error("failed to get or create storage for partition")
return err
}
this.logger.With("partition", ref).Debug("dispatching request")
if !request.Continuous {
_, err := this.executeRead(stream.Context(), partition, request, stream)
return err
}
receiver := partition.Notify(stream.Context())
errDelay := backoff.Exp(1*time.Millisecond, 5*time.Second)
for {
offset, err := this.executeRead(stream.Context(), partition, request, stream)
if err != nil {
if err != io.EOF {
select {
case <-errDelay.DelayC():
continue
case <-stream.Context().Done():
err := stream.Context().Err()
this.logger.WithError(err).Warn("unexpected context done signal")
return err
}
}
} else {
request.Offset = offset
}
errDelay.Reset()
select {
case signalOffset, ok := <-receiver.channel:
this.logger.With("signal_offset", signalOffset).With("ok", ok).Debug("receiver.channel")
if !ok {
this.logger.Debug("signal channel closed")
return nil
}
this.logger.With("offset", signalOffset)
continue
case <-stream.Context().Done():
err := stream.Context().Err()
this.logger.WithError(err).Debug("context done")
return err
}
}
}