Exemple #1
0
func NewTopicPartitionConsumer(host string, topic string, partition int, offset Offset, continuous bool) (*TopicPartitionConsumer, error) {
	connection, err := grpc.Dial(host, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}

	client := api.NewEdgyClient(connection)

	if _, err := client.Ping(context.Background(), &api.PingRequest{}); err != nil {
		connection.Close()
		return nil, err
	}

	consumer := &TopicPartitionConsumer{
		host:       host,
		continuous: continuous,
		offset:     offset,
		logger:     tidy.GetLogger(),
		client:     client,
		topic:      topic,
		partition:  partition,
		messages:   make(chan IncomingBatch),
	}

	go consumer.doReading()
	//go consumer.doDispatching()

	return consumer, nil
}
Exemple #2
0
func NewProducer(cluster Cluster, config ProducerConfig) (*Producer, error) {
	producer := &Producer{
		cluster:  cluster,
		logger:   tidy.GetLogger(),
		requests: make(chan appendRequest),
		config:   config,
	}
	return producer, nil
}
Exemple #3
0
func newPartition(ref PartitionRef, config PartitionConfig, directory string) *Partition {
	return &Partition{
		ref:       ref,
		segments:  NewSegmentList(),
		directory: directory,
		config:    config,
		logger:    tidy.GetLogger().With("partition", ref.String()),
	}
}
Exemple #4
0
func (this *TopicPartitionConsumer) doReading() {
	//delay := backoff.Exp(time.Millisecond, 1*time.Second)

	//defer close(this.dispatch)
	defer func() {
		close(this.messages)
		logger.Debug("batch consumer done")
	}()

	logger := tidy.GetLogger().Withs(tidy.Fields{
		"host":      this.host,
		"topic":     this.topic,
		"partition": this.partition,
		"offset":    this.offset,
	})
	logger.Debug("reading started")

	replies, err := this.client.Read(context.Background(), &api.ReadRequest{
		Topic:      this.topic,
		Partition:  int32(this.partition),
		Offset:     this.offset.toOffsetData(),
		Continuous: this.continuous,
	})

	if err != nil {
		logger.WithError(err).Error("read request failed")
		return
	}

	for {
		reply, err := replies.Recv()
		if err != nil {
			logger.WithError(err).Error("read request failed")
			return
		}

		if len(reply.Messages) == 0 {
			logger.With("offset", this.offset).Warn("EOF")
			return
		}

		if this.logger.IsDebug() {
			logger.With("offset", this.offset).Debug("reply received")
		}

		this.messages <- IncomingBatch{
			Topic:     this.topic,
			Partition: this.partition,
			Offset:    this.offset,
			Messages:  storage.NewMessageSetFromBuffer(reply.Messages),
		}

		this.offset = offsetFromOffsetData(reply.Offset)
	}
}
Exemple #5
0
func NewController(directory string, metricsRegistery metrics.Registry) *Controller {
	controller := &Controller{
		requests:         make(chan RequestContext),
		metricsRegistery: metricsRegistery,
		logger:           tidy.GetLogger(),
		partitions:       make(map[storage.PartitionRef]*PartitionController),
		directory:        directory,
	}
	controller.start(runtime.NumCPU())
	return controller
}
Exemple #6
0
func ListenAndServe(address string, directory string, registry metrics.Registry) error {
	logger := tidy.GetLogger().With("address", address)
	server := &Server{
		registry:   registry,
		logger:     logger,
		controller: NewController(directory, registry),
	}

	logger.Debug("creating listeners")
	listener, err := net.Listen("tcp", address)

	if err != nil {
		logger.With("address", address).WithError(err).Warn("listening failed")
		return err
	}

	logger.Withs(tidy.Fields{
		"address":   address,
		"directory": directory,
	}).Info("serving")

	return server.Serve(listener)
}
Exemple #7
0
func NewPartitionController(ref storage.PartitionRef, rootDir string, metricsRegistry metrics.Registry) *PartitionController {
	controller := &PartitionController{
		bytesInRate:  metrics.GetOrRegisterCounter("bytes-in", metricsRegistry),
		bytesOutRate: metrics.GetOrRegisterCounter("bytes-out", metricsRegistry),

		messagesInRate:  metrics.GetOrRegisterCounter("messages-in", metricsRegistry),
		messagesOutRate: metrics.GetOrRegisterCounter("messages-out", metricsRegistry),

		ref:            ref,
		rootDir:        rootDir,
		logger:         tidy.GetLogger(),
		appendRequests: make(chan *AppendRequest),

		ready: make(chan struct{}),
	}
	go controller.initialize()
	go controller.appendLoop()

	// TODO: remove blocking after read isn't a hack
	<-controller.ready

	return controller
}
Exemple #8
0
func CreateSegment(id SegmentId, filename string, size int64) (*Segment, error) {
	logger := tidy.GetLogger().With("segment", id.String())

	file, err := os.Create(filename)
	if err != nil {
		logger.WithError(err).Withs(tidy.Fields{
			"filename": filename,
			"size":     size,
		}).Debug("segment file creation failed")

		return nil, err
	}

	if err := file.Truncate(size); err != nil {
		logger.WithError(err).Withs(tidy.Fields{
			"filename": filename,
			"size":     size,
		}).Debug("segment file truncation failed")
		file.Close()
		return nil, err
	}

	if logger.IsDebug() {
		logger.Withs(tidy.Fields{
			"filename": filename,
			"size":     size,
		}).Debug("segment created")
	}

	return &Segment{
		filename: file.Name(),
		lock:     new(sync.RWMutex),
		file:     file,
		position: 0,
		size:     size,
	}, nil
}
Exemple #9
0
package storage

import "github.com/pjvds/tidy"

var logger = tidy.GetLogger()

type IndexEntry struct {
	// The id of the message.
	Id MessageId
	// The name of the file that contains the message.
	Filename string
	// The file offset in bytes relative to that file.
	Offset int
	// The length in bytes.
	Length int
}

type Index struct {
	items []IndexEntry
}

func (this *Index) Append(messages *MessageSet) {
	for _, entry := range messages.entries {
		entry := IndexEntry{
			Id:     entry.Id,
			Offset: entry.Offset, // TODO: this should not equal the set offset
			Length: entry.Length,
		}

		this.items = append(this.items, entry)
		logger.With("entry", tidy.Stringify(entry)).Debug("index item appended")