func NewTopicPartitionConsumer(host string, topic string, partition int, offset Offset, continuous bool) (*TopicPartitionConsumer, error) {
connection, err := grpc.Dial(host, grpc.WithInsecure())
if err != nil {
return nil, err
}
client := api.NewEdgyClient(connection)
if _, err := client.Ping(context.Background(), &api.PingRequest{}); err != nil {
connection.Close()
return nil, err
}
consumer := &TopicPartitionConsumer{
host: host,
continuous: continuous,
offset: offset,
logger: tidy.GetLogger(),
client: client,
topic: topic,
partition: partition,
messages: make(chan IncomingBatch),
}
go consumer.doReading()
//go consumer.doDispatching()
return consumer, nil
}
func NewProducer(cluster Cluster, config ProducerConfig) (*Producer, error) {
producer := &Producer{
cluster: cluster,
logger: tidy.GetLogger(),
requests: make(chan appendRequest),
config: config,
}
return producer, nil
}
func newPartition(ref PartitionRef, config PartitionConfig, directory string) *Partition {
return &Partition{
ref: ref,
segments: NewSegmentList(),
directory: directory,
config: config,
logger: tidy.GetLogger().With("partition", ref.String()),
}
}
func (this *TopicPartitionConsumer) doReading() {
//delay := backoff.Exp(time.Millisecond, 1*time.Second)
//defer close(this.dispatch)
defer func() {
close(this.messages)
logger.Debug("batch consumer done")
}()
logger := tidy.GetLogger().Withs(tidy.Fields{
"host": this.host,
"topic": this.topic,
"partition": this.partition,
"offset": this.offset,
})
logger.Debug("reading started")
replies, err := this.client.Read(context.Background(), &api.ReadRequest{
Topic: this.topic,
Partition: int32(this.partition),
Offset: this.offset.toOffsetData(),
Continuous: this.continuous,
})
if err != nil {
logger.WithError(err).Error("read request failed")
return
}
for {
reply, err := replies.Recv()
if err != nil {
logger.WithError(err).Error("read request failed")
return
}
if len(reply.Messages) == 0 {
logger.With("offset", this.offset).Warn("EOF")
return
}
if this.logger.IsDebug() {
logger.With("offset", this.offset).Debug("reply received")
}
this.messages <- IncomingBatch{
Topic: this.topic,
Partition: this.partition,
Offset: this.offset,
Messages: storage.NewMessageSetFromBuffer(reply.Messages),
}
this.offset = offsetFromOffsetData(reply.Offset)
}
}
func NewController(directory string, metricsRegistery metrics.Registry) *Controller {
controller := &Controller{
requests: make(chan RequestContext),
metricsRegistery: metricsRegistery,
logger: tidy.GetLogger(),
partitions: make(map[storage.PartitionRef]*PartitionController),
directory: directory,
}
controller.start(runtime.NumCPU())
return controller
}
func ListenAndServe(address string, directory string, registry metrics.Registry) error {
logger := tidy.GetLogger().With("address", address)
server := &Server{
registry: registry,
logger: logger,
controller: NewController(directory, registry),
}
logger.Debug("creating listeners")
listener, err := net.Listen("tcp", address)
if err != nil {
logger.With("address", address).WithError(err).Warn("listening failed")
return err
}
logger.Withs(tidy.Fields{
"address": address,
"directory": directory,
}).Info("serving")
return server.Serve(listener)
}
func NewPartitionController(ref storage.PartitionRef, rootDir string, metricsRegistry metrics.Registry) *PartitionController {
controller := &PartitionController{
bytesInRate: metrics.GetOrRegisterCounter("bytes-in", metricsRegistry),
bytesOutRate: metrics.GetOrRegisterCounter("bytes-out", metricsRegistry),
messagesInRate: metrics.GetOrRegisterCounter("messages-in", metricsRegistry),
messagesOutRate: metrics.GetOrRegisterCounter("messages-out", metricsRegistry),
ref: ref,
rootDir: rootDir,
logger: tidy.GetLogger(),
appendRequests: make(chan *AppendRequest),
ready: make(chan struct{}),
}
go controller.initialize()
go controller.appendLoop()
// TODO: remove blocking after read isn't a hack
<-controller.ready
return controller
}
func CreateSegment(id SegmentId, filename string, size int64) (*Segment, error) {
logger := tidy.GetLogger().With("segment", id.String())
file, err := os.Create(filename)
if err != nil {
logger.WithError(err).Withs(tidy.Fields{
"filename": filename,
"size": size,
}).Debug("segment file creation failed")
return nil, err
}
if err := file.Truncate(size); err != nil {
logger.WithError(err).Withs(tidy.Fields{
"filename": filename,
"size": size,
}).Debug("segment file truncation failed")
file.Close()
return nil, err
}
if logger.IsDebug() {
logger.Withs(tidy.Fields{
"filename": filename,
"size": size,
}).Debug("segment created")
}
return &Segment{
filename: file.Name(),
lock: new(sync.RWMutex),
file: file,
position: 0,
size: size,
}, nil
}
package storage
import "github.com/pjvds/tidy"
var logger = tidy.GetLogger()
type IndexEntry struct {
// The id of the message.
Id MessageId
// The name of the file that contains the message.
Filename string
// The file offset in bytes relative to that file.
Offset int
// The length in bytes.
Length int
}
type Index struct {
items []IndexEntry
}
func (this *Index) Append(messages *MessageSet) {
for _, entry := range messages.entries {
entry := IndexEntry{
Id: entry.Id,
Offset: entry.Offset, // TODO: this should not equal the set offset
Length: entry.Length,
}
this.items = append(this.items, entry)
logger.With("entry", tidy.Stringify(entry)).Debug("index item appended")