func SmtpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
c.Specify("A SmtpOutput", func() {
smtpOutput := new(SmtpOutput)
config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
config.SendTo = []string{"root"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
runnerName.Return("SmtpOutput")
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
encCall := oth.MockOutputRunner.EXPECT().Encode(pack)
c.Specify("send email payload message", func() {
err := smtpOutput.Init(config)
c.Assume(err, gs.IsNil)
smtpOutput.sendFunction = testSendMail
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
encCall.Return(encoder.Encode(pack))
wg.Add(1)
go func() {
smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait()
})
})
c.Specify("SmtpOutput Message Body Encoding", func() {
smtpOutput := new(SmtpOutput)
chars := "123456789012345678901234567890123456789012345678901234567"
charsE := "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3"
examples := [][]string{
{"Hello", "SGVsbG8="},
{chars, charsE},
{chars + chars, charsE + "\r\n" + charsE},
{chars + chars + "Hello", charsE + "\r\n" + charsE + "\r\n" + "SGVsbG8="},
{"", ""},
{"1", "MQ=="},
}
for _, example := range examples {
smtpOutput.encodeFullMsg([]byte(example[0]))
c.Expect(string(smtpOutput.fullMsg), gs.Equals, example[1])
}
})
// // Use this test with a real server
// c.Specify("Real SmtpOutput output", func() {
// smtpOutput := new(SmtpOutput)
// config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
// config.SendTo = []string{"root"}
// msg := pipeline_ts.GetTestMessage()
// pack := NewPipelinePack(pConfig.InputRecycleChan())
// pack.Message = msg
// pack.Decoded = true
// inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
// inChanCall.Return(inChan)
// runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
// runnerName.Return("SmtpOutput")
// oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
// encCall := oth.MockOutputRunner.EXPECT().Encode(pack)
// c.Specify("send a real email essage", func() {
// err := smtpOutput.Init(config)
// c.Assume(err, gs.IsNil)
// outStr := "Write me out to the network"
// pack.Message.SetPayload(outStr)
// encCall.Return(encoder.Encode(pack))
// go func() {
// wg.Add(1)
// smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
// wg.Done()
// }()
// inChan <- pack
// time.Sleep(1000) // allow time for the message output
// close(inChan)
// wg.Wait()
// // manually check the mail
// })
// })
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockDRunner := NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
c.Specify("with a valid setup and no decoder", func() {
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a message", func() {
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
})
c.Specify("with a valid setup using a decoder", func() {
decoderName := "defaultDecoder"
config.Decoder = decoderName
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
// Mock up our default decoder runner and decoder.
ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
decCall.Return(mockDRunner, true)
mockDecoder := NewMockDecoder(ctrl)
mockDRunner.EXPECT().Decoder().Return(mockDecoder)
c.Specify("consumes a message", func() {
packs := []*PipelinePack{ith.Pack}
mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
c.Specify("consumes a serialized message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the decoded pack
mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
// Ignore leading 5 bytes of encoded message as thats the header
c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close
ug.Add(1)
// Expect the close and the InChan calls
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func UdpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
udpOutput := new(UdpOutput)
config := udpOutput.ConfigStruct().(*UdpOutputConfig)
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(new(plugins.PayloadEncoderConfig))
inChan := make(chan *pipeline.PipelinePack, 1)
rChan := make(chan *pipeline.PipelinePack, 1)
var wg sync.WaitGroup
var rAddr net.Addr
c.Specify("A UdpOutput", func() {
msg := pipeline_ts.GetTestMessage()
payload := "Write me out to the network."
msg.SetPayload(payload)
pack := pipeline.NewPipelinePack(rChan)
pack.Message = msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
c.Specify("using UDP", func() {
addr := "127.0.0.1:45678"
config.Address = addr
ch := make(chan string, 1)
collectData := func() {
conn, err := net.ListenPacket("udp", addr)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = "localhost:12345"
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "udp")
c.Expect(rAddr.String(), gs.Equals, "127.0.0.1:12345")
close(inChan)
wg.Wait()
})
})
c.Specify("using Unix datagrams", func() {
if runtime.GOOS == "windows" {
return
}
testUnixAddr := func() string {
f, err := ioutil.TempFile("", "_heka_test_sock")
c.Assume(err, gs.IsNil)
addr := f.Name()
f.Close()
os.Remove(addr)
return addr
}
config.Address = testUnixAddr()
config.Net = "unixgram"
ch := make(chan string, 1)
var wg sync.WaitGroup
var rAddr net.Addr
collectData := func() {
conn, err := net.ListenPacket("unixgram", config.Address)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
err = os.Remove(config.Address)
var errMsg string
if err != nil {
errMsg = err.Error()
}
ch <- errMsg
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
result = <-ch
c.Expect(result, gs.Equals, "")
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = testUnixAddr()
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "unixgram")
c.Expect(rAddr.String(), gs.Equals, config.LocalAddress)
close(inChan)
wg.Wait()
})
})
})
c.Specify("drop message contents if their size is bigger than allowed UDP datagram size", func() {
huge_msg := pipeline_ts.GetTestMessage()
payload := strings.Repeat("2", 131014)
huge_msg.SetPayload(payload)
huge_pack := pipeline.NewPipelinePack(rChan)
huge_pack.Message = huge_msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(huge_pack).Return(encoder.Encode(huge_pack))
oth.MockOutputRunner.EXPECT().LogError(fmt.Errorf("Message has exceeded allowed UDP data size: 131014 > 65507"))
config.Address = "localhost:12345"
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- huge_pack
close(inChan)
wg.Wait()
})
c.Specify("checks validation of of Maximum message size limit", func() {
config.Address = "localhost:12345"
config.MaxMessageSize = 100
err := udpOutput.Init(config)
c.Assume(err.Error(), gs.Equals, "Maximum message size can't be smaller than 512 bytes.")
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups.
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel.
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return.
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
bytesChan := make(chan []byte, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init.
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Set up relevant mocks.
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.MockSplitterRunner = NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a text message", func() {
// Create a channel to send data to the input. Drop a message on
// there and close the channel.
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "This is a message")
close(streamChan)
err = <-errChan
})
c.Specify("consumes a protobuf encoded message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, string(msgBody))
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
// Expect the close and the InChan calls.
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct())
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("w/ ProtobufEncoder", func() {
encoder := new(ProtobufEncoder)
encoder.Init(nil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
c.Specify("uses framing", func() {
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
})
c.Specify("but not if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.file.Close()
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
if runtime.GOOS != "windows" {
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = filepath.Join(tmpdir, "out.txt")
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
os.RemoveAll(tmpdir)
})
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
subdir := filepath.Join(os.TempDir(), "subdir")
config.Path = filepath.Join(subdir, "out.txt")
err := fileOutput.Init(config)
defer os.RemoveAll(subdir)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(subdir)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
}
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
runtime.Gosched() // Yield so we can overwrite the timerChan.
fileOutput.timerChan = timerChan
}
cleanUp := func() {
close(inChan)
fileOutput.file.Close()
wg.Done()
}
c.Specify("honors flush interval", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case _ = <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
out, err := encoder.Encode(pack2)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}
func NsqOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
var wg sync.WaitGroup
errChan := make(chan error, 1)
defer close(errChan)
c.Specify("A nsq output", func() {
output := new(NsqOutput)
oth := plugins_ts.NewOutputTestHelper(ctrl)
config := output.ConfigStruct().(*NsqOutputConfig)
config.Topic = "test"
startOutput := func() {
wg.Add(1)
go func() {
errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
}
var mockProducer *MockProducer
output.newProducer = func(addr string, config *nsq.Config) (Producer, error) {
mockProducer, _ = NewMockProducer(addr, config)
return mockProducer, nil
}
msg := pipeline_ts.GetTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
c.Specify("requires at least one address", func() {
err := output.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
config.Addresses = []string{"test.com:4150"}
c.Specify("requires an encoder", func() {
err := output.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(nil)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
encoder := new(plugins.PayloadEncoder)
encoder.Init(&plugins.PayloadEncoderConfig{PrefixTs: false})
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
inChan := make(chan *pipeline.PipelinePack, 1)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
c.Specify("publishes a message with the configured topic", func() {
err := output.Init(config)
c.Assume(err, gs.IsNil)
startOutput()
inChan <- pack
transaction := <-mockProducer.respChan
c.Expect(transaction.Error, gs.IsNil)
close(inChan)
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
})
})
})
}
func UnixOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
c.Specify("A UnixOutput", func() {
unixOutput := new(UdpOutput)
config := unixOutput.ConfigStruct().(*UdpOutputConfig)
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(new(plugins.PayloadEncoderConfig))
inChan := make(chan *pipeline.PipelinePack, 1)
rChan := make(chan *pipeline.PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
payload := "Write me out to the network."
msg.SetPayload(payload)
pack := pipeline.NewPipelinePack(rChan)
pack.Message = msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
c.Specify("using UDP", func() {
addr := "@/heka/unix-out"
config.Address = addr
ch := make(chan string, 1)
var wg sync.WaitGroup
var rAddr net.Addr
collectData := func() {
conn, err := net.ListenPacket("unix", addr)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := unixOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = "@/heka/unix-out"
err := unixOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "unix")
c.Expect(rAddr.String(), gs.Equals, "@/heka/unix-out")
close(inChan)
wg.Wait()
})
})
})
}
func TestSendMessage(t *testing.T) {
ctrl := gomock.NewController(t)
broker := sarama.NewMockBroker(t, 2)
defer func() {
broker.Close()
ctrl.Finish()
}()
topic := "test"
globals := DefaultGlobals()
pConfig := NewPipelineConfig(globals)
broker.SetHandlerByMap(map[string]sarama.MockResponse{
"MetadataRequest": sarama.NewMockMetadataResponse(t).
SetBroker(broker.Addr(), broker.BrokerID()).
SetLeader(topic, 0, broker.BrokerID()),
"ProduceRequest": sarama.NewMockProduceResponse(t),
})
ko := new(KafkaOutput)
ko.SetPipelineConfig(pConfig)
config := ko.ConfigStruct().(*KafkaOutputConfig)
config.Addrs = append(config.Addrs, broker.Addr())
config.Topic = topic
err := ko.Init(config)
if err != nil {
t.Fatal(err)
}
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct().(*plugins.PayloadEncoderConfig))
inChan := make(chan *PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
errChan := make(chan error)
startOutput := func() {
go func() {
err := ko.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
}
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
startOutput()
msgcount := atomic.LoadInt64(&ko.processMessageCount)
if msgcount != 0 {
t.Errorf("Invalid starting processMessageCount %d", msgcount)
}
msgcount = atomic.LoadInt64(&ko.processMessageFailures)
if msgcount != 0 {
t.Errorf("Invalid starting processMessageFailures %d", msgcount)
}
inChan <- pack
close(inChan)
err = <-errChan
if err != nil {
t.Errorf("Error running output %s", err)
}
msgcount = atomic.LoadInt64(&ko.processMessageCount)
if msgcount != 1 {
t.Errorf("Invalid ending processMessageCount %d", msgcount)
}
msgcount = atomic.LoadInt64(&ko.processMessageFailures)
if msgcount != 0 {
t.Errorf("Invalid ending processMessageFailures %d", msgcount)
}
}
func IrcOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pipelineConfig := NewPipelineConfig(nil)
outTestHelper := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
errChan := make(chan error, 1)
tickChan := make(chan time.Time)
inChan := make(chan *PipelinePack, 5)
c.Specify("An IrcOutput", func() {
ircOutput := new(IrcOutput)
ircOutput.InitIrcCon = NewMockIrcConn
encoder := new(plugins.PayloadEncoder)
encoder.Init(&plugins.PayloadEncoderConfig{})
config := ircOutput.ConfigStruct().(*IrcOutputConfig)
config.Server = "irc.example.org"
config.Nick = "heka_bot"
config.Ident = "heka"
config.Channels = []string{"#test_channel"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pipelineConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("requires an encoder", func() {
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
outTestHelper.MockOutputRunner.EXPECT().Encoder().Return(nil)
err = ircOutput.Run(outTestHelper.MockOutputRunner, outTestHelper.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
outTestHelper.MockOutputRunner.EXPECT().Ticker().Return(tickChan)
outTestHelper.MockOutputRunner.EXPECT().Encoder().Return(encoder)
outTestHelper.MockOutputRunner.EXPECT().InChan().Return(inChan)
outTestHelper.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes()
startOutput := func() {
wg.Add(1)
go func() {
err := ircOutput.Run(outTestHelper.MockOutputRunner, outTestHelper.MockHelper)
errChan <- err
wg.Done()
}()
<-mockIrcConn.connected
}
c.Specify("sends incoming messages to an irc server", func() {
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
// Send the data
inChan <- pack
// wait for it to arrive
p := <-ircOutput.OutQueue
ircOutput.OutQueue <- p
// once it's arrived send a tick so it gets processed
tickChan <- time.Now()
close(inChan)
wg.Wait()
// Verify we've exited
c.Expect(mockIrcConn.quit, gs.IsTrue)
// verify the message was sent
msgs := mockIrcConn.msgs[ircChan]
c.Expect(len(msgs), gs.Equals, 1)
c.Expect(msgs[0], gs.Equals, string(*msg.Payload))
})
c.Specify("drops messages when outqueue is full", func() {
config.QueueSize = 1
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
outTestHelper.MockOutputRunner.EXPECT().LogError(
ErrOutQueueFull)
outTestHelper.MockOutputRunner.EXPECT().LogError(
fmt.Errorf("%s Channel: %s.", ErrBacklogQueueFull, ircChan))
startOutput()
// Need to wait for callbacks to get registered before calling part.
// If we've called connected, then callbacks have been registered.
// Leave the irc channel so we can fill up the backlog
ircOutput.Conn.Part(ircChan)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)
// Put some data into the inChan
inChan <- pack
// Wait for it to arrive in the OutQueue and put it back
// We *must* do this before sending the tick to avoid data races
p := <-ircOutput.OutQueue
ircOutput.OutQueue <- p
// One tick so that it goes from the OutQueue into the BacklogQueue
tickChan <- time.Now()
// Verify the item is in the backlog queue and put it back
// (It will never leave this loop if the msg never arrives)
queue := ircOutput.BacklogQueues[0]
msg := <-queue
queue <- msg
c.Expect(len(queue), gs.Equals, 1)
// Send another message to fill the OutQueue
inChan <- pack
// Wait for it to arrive, again
p = <-ircOutput.OutQueue
ircOutput.OutQueue <- p
// Send the tick after it's in the OutQueue so we can try processing
// it. This is where we drop it since we cant send, and the
// BacklogQueue is already full.
tickChan <- time.Now()
// Now we want to also cause the OutQueue to drop a message.
// We don't have to wait for it to arrive in the OutQueue like we do above
// since it shouldn't make it there.
inChan <- pack
inChan <- pack
close(inChan)
wg.Wait()
// Verify we've exited
c.Expect(mockIrcConn.quit, gs.IsTrue)
// verify the backlog queue is empty
c.Expect(len(queue), gs.Equals, 0)
// verify that no messages were sent.
msgs := mockIrcConn.msgs[ircChan]
c.Expect(len(msgs), gs.Equals, 0)
})
c.Specify("automatically reconnects on disconnect", func() {
config.TimeBeforeReconnect = 0
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
outTestHelper.MockOutputRunner.EXPECT().LogMessage(DisconnectMsg)
outTestHelper.MockOutputRunner.EXPECT().LogMessage(ReconnectedMsg)
startOutput()
ircOutput.Conn.Disconnect()
// verify we've been disconnected
c.Expect(<-mockIrcConn.connected, gs.IsFalse)
// Wait to become reconnected.
c.Expect(<-mockIrcConn.connected, gs.IsTrue)
close(inChan)
wg.Wait()
c.Expect(mockIrcConn.quit, gs.IsTrue)
})
c.Specify("logs an error and exits from Run() when it cannot reconnect", func() {
config.TimeBeforeReconnect = 0
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
// Explicitly fail to reconnect
mockIrcConn.failReconnect = true
outTestHelper.MockOutputRunner.EXPECT().LogMessage(DisconnectMsg)
outTestHelper.MockOutputRunner.EXPECT().LogError(fmt.Errorf(ErrReconnecting, mockErrFailReconnect))
startOutput()
// Need to wait for callbacks to get registered before we disconnect
ircOutput.Conn.Disconnect()
// verify we've been disconnected
// Run() should return without closing the inChan because the plugin
// automatically cleans up and returns if it fails to reconnect
wg.Wait()
c.Expect(mockIrcConn.quit, gs.IsTrue)
// do this at the end for test cleanup purposes
close(inChan)
})
c.Specify("when kicked from an irc channel", func() {
c.Specify("rejoins when configured to do so", func() {
config.RejoinOnKick = true
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
// trigger a kick event for the irc channel
kick(mockIrcConn, ircChan)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
// quit
close(inChan)
})
c.Specify("doesnt rejoin when it isnt configured to", func() {
config.RejoinOnKick = false // this is already the default
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
// Since this is the only channel we're in, we should get an
// error that there are no channels to join left after being
// kicked
outTestHelper.MockOutputRunner.EXPECT().LogError(ErrNoJoinableChannels)
// trigger a kick for the irc channel
kick(mockIrcConn, ircChan)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
// We shouldnt need to close the inChan, since we have no
// joinable channels, we should be cleaning up already.
})
wg.Wait()
})
c.Specify("when trying to join an unjoinable channel", func() {
c.Specify("it logs an error", func() {
config.Channels = []string{"foo", "baz"}
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 2)
ircChan := ircOutput.Channels[0]
startOutput()
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
reason := "foo"
err = IrcCannotJoinError{ircChan, reason}
outTestHelper.MockOutputRunner.EXPECT().LogError(err)
args := []string{"", ircChan, reason}
event := irc.Event{Arguments: args}
c.Specify("when the channel key is wrong", func() {
event.Code = IRC_ERR_BADCHANNELKEY
})
c.Specify("when banned from the channel", func() {
event.Code = IRC_ERR_BANNEDFROMCHAN
})
c.Specify("when there is no such channel", func() {
event.Code = IRC_NOSUCHCHANNEL
})
c.Specify("when the channel is invite only", func() {
event.Code = IRC_ERR_INVITEONLYCHAN
})
ircOutput.Conn.RunCallbacks(&event)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
// should still be unable to join
ircOutput.Join(ircChan)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
close(inChan)
})
c.Specify("it exits if there are no joinable channels", func() {
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
reason := "foo"
err = IrcCannotJoinError{ircChan, reason}
outTestHelper.MockOutputRunner.EXPECT().LogError(err)
outTestHelper.MockOutputRunner.EXPECT().LogError(ErrNoJoinableChannels)
args := []string{"", ircChan, reason}
event := irc.Event{Arguments: args, Code: IRC_ERR_BADCHANNELKEY}
ircOutput.Conn.RunCallbacks(&event)
// No close since we should exit without.
})
wg.Wait()
})
c.Specify("when trying to join a full channel", func() {
config.TimeBeforeRejoin = 0
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
// We should be able to join before this
ircOutput.Join(ircChan)
reason := "full channel"
err = fmt.Errorf("%s. Retrying in %d seconds.",
IrcCannotJoinError{ircChan, reason}.Error(),
ircOutput.TimeBeforeRejoin)
args := []string{"", ircChan, reason}
event := irc.Event{Code: IRC_ERR_CHANNELISFULL, Arguments: args}
c.Specify("logs an error after using up its attempts", func() {
mockIrcConn.failJoin = true
// We should leave the channel so that we aren't just
// causing a failure to join even when we're already in the
// channel
ircOutput.Conn.Part(ircChan)
maxRetries := int(ircOutput.MaxJoinRetries)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)
gomock.InOrder(
outTestHelper.MockOutputRunner.EXPECT().LogError(
err).Times(maxRetries),
outTestHelper.MockOutputRunner.EXPECT().LogError(
fmt.Errorf(ErrUsedUpRetryAttempts, ircChan)),
outTestHelper.MockOutputRunner.EXPECT().LogError(
ErrNoJoinableChannels),
)
for i := 1; i <= maxRetries; i++ {
// Trigger the event the number of times
ircOutput.Conn.RunCallbacks(&event)
c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, i)
}
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)
ircOutput.Conn.RunCallbacks(&event)
// We've used up our attempts,
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
ircOutput.Conn.RunCallbacks(&event)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
})
c.Specify("resets its attempts when it successful", func() {
mockIrcConn.failJoin = true
outTestHelper.MockOutputRunner.EXPECT().LogError(err).Times(2)
ircOutput.Conn.RunCallbacks(&event)
c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, 1)
mockIrcConn.failJoin = false
ircOutput.Conn.RunCallbacks(&event)
c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, 0)
})
close(inChan)
wg.Wait()
})
c.Specify("when banned from a server logs an error and quits", func() {
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
startOutput()
event := irc.Event{Code: IRC_ERR_YOUREBANNEDCREEP}
outTestHelper.MockOutputRunner.EXPECT().LogError(ErrBannedFromServer)
ircOutput.Conn.RunCallbacks(&event)
// We should be able to exit without closing the channel first
wg.Wait()
// Clean this up
close(inChan)
})
c.Expect(<-errChan, gs.IsNil)
c.Expect(<-mockIrcConn.connected, gs.IsFalse)
})
// Cleanup which should happen each run
close(mockIrcConn.connected)
close(mockIrcConn.delivered)
close(mockIrcConn.Error)
})
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct())
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.QueueCursor = "queuecursor"
errChan := make(chan error, 1)
c.Specify("w/ ProtobufEncoder", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
c.Specify("uses framing", func() {
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
})
c.Specify("but not if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
})
c.Specify("rotates files correctly", func() {
config.Path = "%Y-%m-%d"
config.RotationInterval = 24
rotateChan := make(chan time.Time)
closingChan := make(chan struct{})
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
defer fileOutput.file.Close()
fileOutput.rotateChan = rotateChan
fileOutput.closing = closingChan
fileOutput.startRotateNotifier()
committerChan := make(chan struct{})
go func() {
fileOutput.committer(oth.MockOutputRunner, errChan)
close(committerChan)
}()
c.Assume(fileOutput.path, gs.Equals, time.Now().Format("2006-01-02"))
futureDuration, _ := time.ParseDuration("24h")
futureNow := time.Now().Add(futureDuration)
rotateChan <- futureNow
close(inChan)
close(fileOutput.batchChan)
<-committerChan
c.Assume(fileOutput.path, gs.Equals, futureNow.Format("2006-01-02"))
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.file.Close()
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
go fileOutput.receiver(oth.MockOutputRunner, errChan)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
c.Expect(string(outBatch.data), gs.Equals, payload)
c.Expect(outBatch.cursor, gs.Equals, pack.QueueCursor)
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
batch := &outBatch{
data: outBytes,
cursor: pack.QueueCursor,
}
oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop.
go fileOutput.committer(oth.MockOutputRunner, errChan)
// Feed and close the batchChan.
go func() {
fileOutput.batchChan <- batch
_ = <-fileOutput.backChan // clear backChan to prevent blocking.
close(fileOutput.batchChan)
}()
// Wait until we know processing has finished.
<-fileOutput.closing
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop.
go fileOutput.committer(oth.MockOutputRunner, errChan)
// Feed and close the batchChan.
go func() {
fileOutput.batchChan <- batch
_ = <-fileOutput.backChan // clear backChan to prevent blocking.
close(fileOutput.batchChan)
}()
// Wait until we know processing has finished.
<-fileOutput.closing
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other.
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
if runtime.GOOS != "windows" {
if u, err := user.Current(); err != nil && u.Uid != "0" {
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = filepath.Join(tmpdir, "out.txt")
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
os.RemoveAll(tmpdir)
})
}
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
subdir := filepath.Join(os.TempDir(), "subdir")
config.Path = filepath.Join(subdir, "out.txt")
err := fileOutput.Init(config)
defer os.RemoveAll(subdir)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(subdir)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
}
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
msg2.SetPayload("MESSAGE 2")
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.timerChan = timerChan
go fileOutput.receiver(oth.MockOutputRunner, errChan)
runtime.Gosched() // Yield so receiver will start.
}
cleanUp := func() {
close(inChan)
fileOutput.file.Close()
}
c.Specify("honors flush interval", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case _ = <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
after = time.After(100 * time.Millisecond)
inChan <- pack2
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
out, err := encoder.Encode(pack2)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
inChan <- pack2
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}