コード例 #1
0
ファイル: producer.go プロジェクト: tehol/kafka-pixy
// Spawn creates a producer instance and starts its internal goroutines.
func Spawn(cfg *config.T) (*T, error) {
	saramaCfg := sarama.NewConfig()
	saramaCfg.ChannelBufferSize = cfg.Producer.ChannelBufferSize
	saramaCfg.Producer.RequiredAcks = sarama.WaitForAll
	saramaCfg.Producer.Return.Successes = true
	saramaCfg.Producer.Return.Errors = true
	saramaCfg.Producer.Compression = sarama.CompressionSnappy
	saramaCfg.Producer.Retry.Backoff = 10 * time.Second
	saramaCfg.Producer.Retry.Max = 6
	saramaCfg.Producer.Flush.Frequency = 500 * time.Millisecond
	saramaCfg.Producer.Flush.Bytes = 1024 * 1024

	saramaClient, err := sarama.NewClient(cfg.Kafka.SeedPeers, saramaCfg)
	if err != nil {
		return nil, fmt.Errorf("failed to create sarama.Client, err=(%s)", err)
	}
	saramaProducer, err := sarama.NewAsyncProducerFromClient(saramaClient)
	if err != nil {
		return nil, fmt.Errorf("failed to create sarama.Producer, err=(%s)", err)
	}

	p := &T{
		baseCID:         sarama.RootCID.NewChild("producer"),
		saramaClient:    saramaClient,
		saramaProducer:  saramaProducer,
		shutdownTimeout: cfg.Producer.ShutdownTimeout,
		deadMessageCh:   cfg.Producer.DeadMessageCh,
		dispatcherCh:    make(chan *sarama.ProducerMessage, cfg.Producer.ChannelBufferSize),
		resultCh:        make(chan produceResult, cfg.Producer.ChannelBufferSize),
	}
	spawn(&p.wg, p.merge)
	spawn(&p.wg, p.dispatch)
	return p, nil
}
コード例 #2
0
ファイル: testhelpers.go プロジェクト: tehol/kafka-pixy
func NewKafkaHelper(c *C) *KafkaHelper {
	kh := &KafkaHelper{c: c}
	cfg := sarama.NewConfig()
	cfg.Producer.Return.Successes = true
	cfg.Producer.Return.Errors = true
	cfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
	cfg.ClientID = "unittest-runner"
	err := error(nil)
	if kh.client, err = sarama.NewClient(KafkaPeers, cfg); err != nil {
		panic(err)
	}
	if kh.consumer, err = sarama.NewConsumerFromClient(kh.client); err != nil {
		panic(err)
	}
	if kh.producer, err = sarama.NewAsyncProducerFromClient(kh.client); err != nil {
		panic(err)
	}
	if kh.offsetMgr, err = sarama.NewOffsetManagerFromClient(kh.client); err != nil {
		panic(err)
	}
	return kh
}
コード例 #3
0
ファイル: producer.go プロジェクト: lucmichalski/kafka-pixy
// SpawnGracefulProducer creates a `KafkaProducer` instance and starts its internal
// goroutines.
func SpawnGracefulProducer(config *config.T) (*GracefulProducer, error) {
	saramaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.SaramaConfig())
	if err != nil {
		return nil, fmt.Errorf("failed to create sarama.Client, err=(%s)", err)
	}
	saramaProducer, err := sarama.NewAsyncProducerFromClient(saramaClient)
	if err != nil {
		return nil, fmt.Errorf("failed to create sarama.Producer, err=(%s)", err)
	}
	gp := &GracefulProducer{
		baseCID:         sarama.RootCID.NewChild("producer"),
		saramaClient:    saramaClient,
		saramaProducer:  saramaProducer,
		shutdownTimeout: config.Producer.ShutdownTimeout,
		deadMessageCh:   config.Producer.DeadMessageCh,
		dispatcherCh:    make(chan *sarama.ProducerMessage, config.ChannelBufferSize),
		resultCh:        make(chan produceResult, config.ChannelBufferSize),
	}
	spawn(&gp.wg, gp.merge)
	spawn(&gp.wg, gp.dispatch)
	return gp, nil
}