func TestProducerConfigMessageMaxBytes(t *testing.T) {
cp := newChainPartition(provisional.TestChainID, rawPartition)
broker := sarama.NewMockBroker(t, 1)
defer func() {
broker.Close()
}()
broker.SetHandlerByMap(map[string]sarama.MockResponse{
"MetadataRequest": sarama.NewMockMetadataResponse(t).
SetBroker(broker.Addr(), broker.BrokerID()).
SetLeader(cp.Topic(), cp.Partition(), broker.BrokerID()),
"ProduceRequest": sarama.NewMockProduceResponse(t),
})
config := newBrokerConfig(testConf.Kafka.Version, rawPartition)
producer, err := sarama.NewSyncProducer([]string{broker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
defer func() {
producer.Close()
}()
testCases := []struct {
name string
size int
err error
}{
{"TypicalDeploy", 8 * 1024 * 1024, nil},
{"TooBig", 100*1024*1024 + 1, sarama.ErrMessageSizeTooLarge},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
_, _, err = producer.SendMessage(&sarama.ProducerMessage{
Topic: cp.Topic(),
Value: sarama.ByteEncoder(make([]byte, tc.size)),
})
if err != tc.err {
t.Fatal(err)
}
})
}
}
func TestNewBrokerConfig(t *testing.T) {
// Use a partition ID that is not the 'default' (rawPartition)
var differentPartition int32 = 2
cp = newChainPartition(provisional.TestChainID, differentPartition)
// Setup a mock broker that reports that it has 3 partitions for the topic
broker := sarama.NewMockBroker(t, 1)
defer func() {
broker.Close()
}()
broker.SetHandlerByMap(map[string]sarama.MockResponse{
"MetadataRequest": sarama.NewMockMetadataResponse(t).
SetBroker(broker.Addr(), broker.BrokerID()).
SetLeader(cp.Topic(), 0, broker.BrokerID()).
SetLeader(cp.Topic(), 1, broker.BrokerID()).
SetLeader(cp.Topic(), 2, broker.BrokerID()),
"ProduceRequest": sarama.NewMockProduceResponse(t),
})
config := newBrokerConfig(testConf.Kafka.Version, differentPartition)
producer, err := sarama.NewSyncProducer([]string{broker.Addr()}, config)
if err != nil {
t.Fatal("Failed to create producer:", err)
}
defer func() {
producer.Close()
}()
for i := 0; i < 10; i++ {
assignedPartition, _, err := producer.SendMessage(&sarama.ProducerMessage{Topic: cp.Topic()})
if err != nil {
t.Fatal("Failed to send message:", err)
}
if assignedPartition != differentPartition {
t.Fatalf("Message wasn't posted to the right partition - expected: %d, got %v", differentPartition, assignedPartition)
}
}
}
func TestSendMessage(t *testing.T) {
ctrl := gomock.NewController(t)
broker := sarama.NewMockBroker(t, 2)
defer func() {
broker.Close()
ctrl.Finish()
}()
topic := "test"
globals := DefaultGlobals()
pConfig := NewPipelineConfig(globals)
broker.SetHandlerByMap(map[string]sarama.MockResponse{
"MetadataRequest": sarama.NewMockMetadataResponse(t).
SetBroker(broker.Addr(), broker.BrokerID()).
SetLeader(topic, 0, broker.BrokerID()),
"ProduceRequest": sarama.NewMockProduceResponse(t),
})
ko := new(KafkaOutput)
ko.SetPipelineConfig(pConfig)
config := ko.ConfigStruct().(*KafkaOutputConfig)
config.Addrs = append(config.Addrs, broker.Addr())
config.Topic = topic
err := ko.Init(config)
if err != nil {
t.Fatal(err)
}
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct().(*plugins.PayloadEncoderConfig))
inChan := make(chan *PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
errChan := make(chan error)
startOutput := func() {
go func() {
err := ko.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
}
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
startOutput()
msgcount := atomic.LoadInt64(&ko.processMessageCount)
if msgcount != 0 {
t.Errorf("Invalid starting processMessageCount %d", msgcount)
}
msgcount = atomic.LoadInt64(&ko.processMessageFailures)
if msgcount != 0 {
t.Errorf("Invalid starting processMessageFailures %d", msgcount)
}
inChan <- pack
close(inChan)
err = <-errChan
if err != nil {
t.Errorf("Error running output %s", err)
}
msgcount = atomic.LoadInt64(&ko.processMessageCount)
if msgcount != 1 {
t.Errorf("Invalid ending processMessageCount %d", msgcount)
}
msgcount = atomic.LoadInt64(&ko.processMessageFailures)
if msgcount != 0 {
t.Errorf("Invalid ending processMessageFailures %d", msgcount)
}
}