Exemple #1
0
func TestProducerConfigMessageMaxBytes(t *testing.T) {
	cp := newChainPartition(provisional.TestChainID, rawPartition)

	broker := sarama.NewMockBroker(t, 1)
	defer func() {
		broker.Close()
	}()
	broker.SetHandlerByMap(map[string]sarama.MockResponse{
		"MetadataRequest": sarama.NewMockMetadataResponse(t).
			SetBroker(broker.Addr(), broker.BrokerID()).
			SetLeader(cp.Topic(), cp.Partition(), broker.BrokerID()),
		"ProduceRequest": sarama.NewMockProduceResponse(t),
	})

	config := newBrokerConfig(testConf.Kafka.Version, rawPartition)
	producer, err := sarama.NewSyncProducer([]string{broker.Addr()}, config)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		producer.Close()
	}()

	testCases := []struct {
		name string
		size int
		err  error
	}{
		{"TypicalDeploy", 8 * 1024 * 1024, nil},
		{"TooBig", 100*1024*1024 + 1, sarama.ErrMessageSizeTooLarge},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			_, _, err = producer.SendMessage(&sarama.ProducerMessage{
				Topic: cp.Topic(),
				Value: sarama.ByteEncoder(make([]byte, tc.size)),
			})
			if err != tc.err {
				t.Fatal(err)
			}
		})

	}
}
Exemple #2
0
func TestNewBrokerConfig(t *testing.T) {
	// Use a partition ID that is not the 'default' (rawPartition)
	var differentPartition int32 = 2
	cp = newChainPartition(provisional.TestChainID, differentPartition)

	// Setup a mock broker that reports that it has 3 partitions for the topic
	broker := sarama.NewMockBroker(t, 1)
	defer func() {
		broker.Close()
	}()
	broker.SetHandlerByMap(map[string]sarama.MockResponse{
		"MetadataRequest": sarama.NewMockMetadataResponse(t).
			SetBroker(broker.Addr(), broker.BrokerID()).
			SetLeader(cp.Topic(), 0, broker.BrokerID()).
			SetLeader(cp.Topic(), 1, broker.BrokerID()).
			SetLeader(cp.Topic(), 2, broker.BrokerID()),
		"ProduceRequest": sarama.NewMockProduceResponse(t),
	})

	config := newBrokerConfig(testConf.Kafka.Version, differentPartition)
	producer, err := sarama.NewSyncProducer([]string{broker.Addr()}, config)
	if err != nil {
		t.Fatal("Failed to create producer:", err)
	}
	defer func() {
		producer.Close()
	}()

	for i := 0; i < 10; i++ {
		assignedPartition, _, err := producer.SendMessage(&sarama.ProducerMessage{Topic: cp.Topic()})
		if err != nil {
			t.Fatal("Failed to send message:", err)
		}
		if assignedPartition != differentPartition {
			t.Fatalf("Message wasn't posted to the right partition - expected: %d, got %v", differentPartition, assignedPartition)
		}
	}
}
func TestSendMessage(t *testing.T) {
	ctrl := gomock.NewController(t)
	broker := sarama.NewMockBroker(t, 2)

	defer func() {
		broker.Close()
		ctrl.Finish()
	}()

	topic := "test"
	globals := DefaultGlobals()
	pConfig := NewPipelineConfig(globals)

	broker.SetHandlerByMap(map[string]sarama.MockResponse{
		"MetadataRequest": sarama.NewMockMetadataResponse(t).
			SetBroker(broker.Addr(), broker.BrokerID()).
			SetLeader(topic, 0, broker.BrokerID()),
		"ProduceRequest": sarama.NewMockProduceResponse(t),
	})

	ko := new(KafkaOutput)
	ko.SetPipelineConfig(pConfig)
	config := ko.ConfigStruct().(*KafkaOutputConfig)
	config.Addrs = append(config.Addrs, broker.Addr())
	config.Topic = topic
	err := ko.Init(config)
	if err != nil {
		t.Fatal(err)
	}
	oth := plugins_ts.NewOutputTestHelper(ctrl)
	encoder := new(plugins.PayloadEncoder)
	encoder.Init(encoder.ConfigStruct().(*plugins.PayloadEncoderConfig))

	inChan := make(chan *PipelinePack, 1)

	msg := pipeline_ts.GetTestMessage()
	pack := NewPipelinePack(pConfig.InputRecycleChan())
	pack.Message = msg

	inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
	inChanCall.Return(inChan)

	errChan := make(chan error)
	startOutput := func() {
		go func() {
			err := ko.Run(oth.MockOutputRunner, oth.MockHelper)
			errChan <- err
		}()
	}

	oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
	oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))

	outStr := "Write me out to the network"
	pack.Message.SetPayload(outStr)
	startOutput()

	msgcount := atomic.LoadInt64(&ko.processMessageCount)
	if msgcount != 0 {
		t.Errorf("Invalid starting processMessageCount %d", msgcount)
	}
	msgcount = atomic.LoadInt64(&ko.processMessageFailures)
	if msgcount != 0 {
		t.Errorf("Invalid starting processMessageFailures %d", msgcount)
	}

	inChan <- pack
	close(inChan)
	err = <-errChan
	if err != nil {
		t.Errorf("Error running output %s", err)
	}

	msgcount = atomic.LoadInt64(&ko.processMessageCount)
	if msgcount != 1 {
		t.Errorf("Invalid ending processMessageCount %d", msgcount)
	}
	msgcount = atomic.LoadInt64(&ko.processMessageFailures)
	if msgcount != 0 {
		t.Errorf("Invalid ending processMessageFailures %d", msgcount)
	}
}