func DashboardOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
dashboardOutput := new(DashboardOutput)
dashboardOutput.pConfig = pConfig
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)
errChan := make(chan error, 1)
startOutput := func() {
go func() {
errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
}()
}
if runtime.GOOS != "windows" {
c.Specify("A DashboardOutput", func() {
tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
c.Assume(err, gs.IsNil)
config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
config.WorkingDirectory = tmpdir
c.Specify("Init halts if basedirectory is not writable", func() {
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
err = dashboardOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("that is running", func() {
startedChan := make(chan bool, 1)
defer close(startedChan)
ts := httptest.NewUnstartedServer(nil)
dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
ts.Start()
startedChan <- true
return nil
}
ticker := make(chan time.Time)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)
err := os.MkdirAll(tmpdir, 0700)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Noop
})
c.Specify("sets custom http headers", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err = dashboardOutput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = dashboardOutput.server
startOutput()
inChan <- pack
<-startedChan
resp, err := http.Get(ts.URL)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
// Verify headers are there
eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
c.Expect(eq, gs.IsTrue)
eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
c.Expect(eq, gs.IsTrue)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
ts.Close()
})
})
}
}
func UdpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
udpOutput := new(UdpOutput)
config := udpOutput.ConfigStruct().(*UdpOutputConfig)
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(new(plugins.PayloadEncoderConfig))
inChan := make(chan *pipeline.PipelinePack, 1)
rChan := make(chan *pipeline.PipelinePack, 1)
var wg sync.WaitGroup
var rAddr net.Addr
c.Specify("A UdpOutput", func() {
msg := pipeline_ts.GetTestMessage()
payload := "Write me out to the network."
msg.SetPayload(payload)
pack := pipeline.NewPipelinePack(rChan)
pack.Message = msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
c.Specify("using UDP", func() {
addr := "127.0.0.1:45678"
config.Address = addr
ch := make(chan string, 1)
collectData := func() {
conn, err := net.ListenPacket("udp", addr)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = "localhost:12345"
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "udp")
c.Expect(rAddr.String(), gs.Equals, "127.0.0.1:12345")
close(inChan)
wg.Wait()
})
})
c.Specify("using Unix datagrams", func() {
if runtime.GOOS == "windows" {
return
}
testUnixAddr := func() string {
f, err := ioutil.TempFile("", "_heka_test_sock")
c.Assume(err, gs.IsNil)
addr := f.Name()
f.Close()
os.Remove(addr)
return addr
}
config.Address = testUnixAddr()
config.Net = "unixgram"
ch := make(chan string, 1)
var wg sync.WaitGroup
var rAddr net.Addr
collectData := func() {
conn, err := net.ListenPacket("unixgram", config.Address)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
err = os.Remove(config.Address)
var errMsg string
if err != nil {
errMsg = err.Error()
}
ch <- errMsg
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
result = <-ch
c.Expect(result, gs.Equals, "")
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = testUnixAddr()
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "unixgram")
c.Expect(rAddr.String(), gs.Equals, config.LocalAddress)
close(inChan)
wg.Wait()
})
})
})
c.Specify("drop message contents if their size is bigger than allowed UDP datagram size", func() {
huge_msg := pipeline_ts.GetTestMessage()
payload := strings.Repeat("2", 131014)
huge_msg.SetPayload(payload)
huge_pack := pipeline.NewPipelinePack(rChan)
huge_pack.Message = huge_msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(huge_pack).Return(encoder.Encode(huge_pack))
oth.MockOutputRunner.EXPECT().LogError(fmt.Errorf("Message has exceeded allowed UDP data size: 131014 > 65507"))
config.Address = "localhost:12345"
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- huge_pack
close(inChan)
wg.Wait()
})
c.Specify("checks validation of of Maximum message size limit", func() {
config.Address = "localhost:12345"
config.MaxMessageSize = 100
err := udpOutput.Init(config)
c.Assume(err.Error(), gs.Equals, "Maximum message size can't be smaller than 512 bytes.")
})
}
func SmtpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
c.Specify("A SmtpOutput", func() {
smtpOutput := new(SmtpOutput)
config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
config.SendTo = []string{"root"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
runnerName.Return("SmtpOutput")
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()
c.Specify("send email payload message", func() {
err := smtpOutput.Init(config)
c.Assume(err, gs.IsNil)
smtpOutput.sendFunction = testSendMail
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
go func() {
wg.Add(1)
smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait()
})
})
// Use this test with a real server
// c.Specify("Real SmtpOutput output", func() {
// smtpOutput := new(SmtpOutput)
//
// config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
// config.SendTo = []string{"root"}
//
// msg := pipeline_ts.GetTestMessage()
// pack := NewPipelinePack(pConfig.InputRecycleChan())
// pack.Message = msg
// pack.Decoded = true
// inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
// inChanCall.Return(inChan)
// runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
// runnerName.Return("SmtpOutput")
// oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()
//
// c.Specify("send a real email essage", func() {
//
// err := smtpOutput.Init(config)
// c.Assume(err, gs.IsNil)
//
// outStr := "Write me out to the network"
// pack.Message.SetPayload(outStr)
// go func() {
// wg.Add(1)
// smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
// wg.Done()
// }()
// inChan <- pack
// time.Sleep(1000) // allow time for the message output
// close(inChan)
// wg.Wait()
// // manually check the mail
// })
// })
}
func TcpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests")
defer func() {
ctrl.Finish()
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
pConfig := NewPipelineConfig(nil)
c.Specify("TcpOutput", func() {
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
}()
tcpOutput := new(TcpOutput)
tcpOutput.SetName("test")
config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
tcpOutput.Init(config)
tickChan := make(chan time.Time)
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan)
encoder := new(ProtobufEncoder)
encoder.Init(nil)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
outStr := "Write me out to the network"
newpack := NewPipelinePack(nil)
newpack.Message = msg
newpack.Decoded = true
newpack.Message.SetPayload(outStr)
matchBytes, err := proto.Marshal(newpack.Message)
c.Expect(err, gs.IsNil)
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
c.Specify("doesn't use framing w/o ProtobufEncoder", func() {
encoder := new(plugins.PayloadEncoder)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
c.Specify("doesn't use framing if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
c.Specify("writes out to the network", func() {
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch // wait for server
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()
pack.Message.SetPayload(outStr)
wg.Add(1)
go func() {
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
inChan <- pack
result = <-ch
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(1))
c.Expect(result, gs.Equals, string(matchBytes))
close(inChan)
wg.Wait() // wait for output to finish shutting down
})
c.Specify("far end not initially listening", func() {
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
pack.Message.SetPayload(outStr)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()
wg.Add(1)
go func() {
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
inChan <- pack
for x := 0; x < 5 && msgcount == 0; x++ {
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
time.Sleep(time.Duration(100) * time.Millisecond)
}
// After the message is queued start the collector. However, we
// don't have a way guarantee a send attempt has already been made
// and that we are actually exercising the retry code.
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch
c.Expect(result, gs.Equals, string(matchBytes))
close(inChan)
wg.Wait() // wait for output to finish shutting down
})
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockDRunner := NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
c.Specify("with a valid setup and no decoder", func() {
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a message", func() {
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
})
c.Specify("with a valid setup using a decoder", func() {
decoderName := "defaultDecoder"
config.Decoder = decoderName
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
// Mock up our default decoder runner and decoder.
ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
decCall.Return(mockDRunner, true)
mockDecoder := NewMockDecoder(ctrl)
mockDRunner.EXPECT().Decoder().Return(mockDecoder)
c.Specify("consumes a message", func() {
packs := []*PipelinePack{ith.Pack}
mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
c.Specify("consumes a serialized message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the decoded pack
mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
// Ignore leading 5 bytes of encoded message as thats the header
c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close
ug.Add(1)
// Expect the close and the InChan calls
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func IrcOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pipelineConfig := NewPipelineConfig(nil)
outTestHelper := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
errChan := make(chan error, 1)
tickChan := make(chan time.Time)
inChan := make(chan *PipelinePack, 5)
c.Specify("An IrcOutput", func() {
ircOutput := new(IrcOutput)
ircOutput.InitIrcCon = NewMockIrcConn
encoder := new(plugins.PayloadEncoder)
encoder.Init(&plugins.PayloadEncoderConfig{})
config := ircOutput.ConfigStruct().(*IrcOutputConfig)
config.Server = "irc.example.org"
config.Nick = "heka_bot"
config.Ident = "heka"
config.Channels = []string{"#test_channel"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pipelineConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("requires an encoder", func() {
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
outTestHelper.MockOutputRunner.EXPECT().Encoder().Return(nil)
err = ircOutput.Run(outTestHelper.MockOutputRunner, outTestHelper.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
outTestHelper.MockOutputRunner.EXPECT().Ticker().Return(tickChan)
outTestHelper.MockOutputRunner.EXPECT().Encoder().Return(encoder)
outTestHelper.MockOutputRunner.EXPECT().InChan().Return(inChan)
outTestHelper.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes()
startOutput := func() {
wg.Add(1)
go func() {
err := ircOutput.Run(outTestHelper.MockOutputRunner, outTestHelper.MockHelper)
errChan <- err
wg.Done()
}()
<-mockIrcConn.connected
}
c.Specify("sends incoming messages to an irc server", func() {
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
// Send the data
inChan <- pack
// wait for it to arrive
p := <-ircOutput.OutQueue
ircOutput.OutQueue <- p
// once it's arrived send a tick so it gets processed
tickChan <- time.Now()
close(inChan)
wg.Wait()
// Verify we've exited
c.Expect(mockIrcConn.quit, gs.IsTrue)
// verify the message was sent
msgs := mockIrcConn.msgs[ircChan]
c.Expect(len(msgs), gs.Equals, 1)
c.Expect(msgs[0], gs.Equals, string(*msg.Payload))
})
c.Specify("drops messages when outqueue is full", func() {
config.QueueSize = 1
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
outTestHelper.MockOutputRunner.EXPECT().LogError(
ErrOutQueueFull)
outTestHelper.MockOutputRunner.EXPECT().LogError(
fmt.Errorf("%s Channel: %s.", ErrBacklogQueueFull, ircChan))
startOutput()
// Need to wait for callbacks to get registered before calling part.
// If we've called connected, then callbacks have been registered.
// Leave the irc channel so we can fill up the backlog
ircOutput.Conn.Part(ircChan)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)
// Put some data into the inChan
inChan <- pack
// Wait for it to arrive in the OutQueue and put it back
// We *must* do this before sending the tick to avoid data races
p := <-ircOutput.OutQueue
ircOutput.OutQueue <- p
// One tick so that it goes from the OutQueue into the BacklogQueue
tickChan <- time.Now()
// Verify the item is in the backlog queue and put it back
// (It will never leave this loop if the msg never arrives)
queue := ircOutput.BacklogQueues[0]
msg := <-queue
queue <- msg
c.Expect(len(queue), gs.Equals, 1)
// Send another message to fill the OutQueue
inChan <- pack
// Wait for it to arrive, again
p = <-ircOutput.OutQueue
ircOutput.OutQueue <- p
// Send the tick after it's in the OutQueue so we can try processing
// it. This is where we drop it since we cant send, and the
// BacklogQueue is already full.
tickChan <- time.Now()
// Now we want to also cause the OutQueue to drop a message.
// We don't have to wait for it to arrive in the OutQueue like we do above
// since it shouldn't make it there.
inChan <- pack
inChan <- pack
close(inChan)
wg.Wait()
// Verify we've exited
c.Expect(mockIrcConn.quit, gs.IsTrue)
// verify the backlog queue is empty
c.Expect(len(queue), gs.Equals, 0)
// verify that no messages were sent.
msgs := mockIrcConn.msgs[ircChan]
c.Expect(len(msgs), gs.Equals, 0)
})
c.Specify("automatically reconnects on disconnect", func() {
config.TimeBeforeReconnect = 0
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
outTestHelper.MockOutputRunner.EXPECT().LogMessage(DisconnectMsg)
outTestHelper.MockOutputRunner.EXPECT().LogMessage(ReconnectedMsg)
startOutput()
ircOutput.Conn.Disconnect()
// verify we've been disconnected
c.Expect(<-mockIrcConn.connected, gs.IsFalse)
// Wait to become reconnected.
c.Expect(<-mockIrcConn.connected, gs.IsTrue)
close(inChan)
wg.Wait()
c.Expect(mockIrcConn.quit, gs.IsTrue)
})
c.Specify("logs an error and exits from Run() when it cannot reconnect", func() {
config.TimeBeforeReconnect = 0
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
// Explicitly fail to reconnect
mockIrcConn.failReconnect = true
outTestHelper.MockOutputRunner.EXPECT().LogMessage(DisconnectMsg)
outTestHelper.MockOutputRunner.EXPECT().LogError(fmt.Errorf(ErrReconnecting, mockErrFailReconnect))
startOutput()
// Need to wait for callbacks to get registered before we disconnect
ircOutput.Conn.Disconnect()
// verify we've been disconnected
// Run() should return without closing the inChan because the plugin
// automatically cleans up and returns if it fails to reconnect
wg.Wait()
c.Expect(mockIrcConn.quit, gs.IsTrue)
// do this at the end for test cleanup purposes
close(inChan)
})
c.Specify("when kicked from an irc channel", func() {
c.Specify("rejoins when configured to do so", func() {
config.RejoinOnKick = true
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
// trigger a kick event for the irc channel
kick(mockIrcConn, ircChan)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
// quit
close(inChan)
})
c.Specify("doesnt rejoin when it isnt configured to", func() {
config.RejoinOnKick = false // this is already the default
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
// Since this is the only channel we're in, we should get an
// error that there are no channels to join left after being
// kicked
outTestHelper.MockOutputRunner.EXPECT().LogError(ErrNoJoinableChannels)
// trigger a kick for the irc channel
kick(mockIrcConn, ircChan)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
// We shouldnt need to close the inChan, since we have no
// joinable channels, we should be cleaning up already.
})
wg.Wait()
})
c.Specify("when trying to join an unjoinable channel", func() {
c.Specify("it logs an error", func() {
config.Channels = []string{"foo", "baz"}
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 2)
ircChan := ircOutput.Channels[0]
startOutput()
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
reason := "foo"
err = IrcCannotJoinError{ircChan, reason}
outTestHelper.MockOutputRunner.EXPECT().LogError(err)
args := []string{"", ircChan, reason}
event := irc.Event{Arguments: args}
c.Specify("when the channel key is wrong", func() {
event.Code = IRC_ERR_BADCHANNELKEY
})
c.Specify("when banned from the channel", func() {
event.Code = IRC_ERR_BANNEDFROMCHAN
})
c.Specify("when there is no such channel", func() {
event.Code = IRC_NOSUCHCHANNEL
})
c.Specify("when the channel is invite only", func() {
event.Code = IRC_ERR_INVITEONLYCHAN
})
ircOutput.Conn.RunCallbacks(&event)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
// should still be unable to join
ircOutput.Join(ircChan)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
close(inChan)
})
c.Specify("it exits if there are no joinable channels", func() {
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
reason := "foo"
err = IrcCannotJoinError{ircChan, reason}
outTestHelper.MockOutputRunner.EXPECT().LogError(err)
outTestHelper.MockOutputRunner.EXPECT().LogError(ErrNoJoinableChannels)
args := []string{"", ircChan, reason}
event := irc.Event{Arguments: args, Code: IRC_ERR_BADCHANNELKEY}
ircOutput.Conn.RunCallbacks(&event)
// No close since we should exit without.
})
wg.Wait()
})
c.Specify("when trying to join a full channel", func() {
config.TimeBeforeRejoin = 0
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Assume(len(ircOutput.Channels), gs.Equals, 1)
ircChan := ircOutput.Channels[0]
startOutput()
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
// We should be able to join before this
ircOutput.Join(ircChan)
reason := "full channel"
err = fmt.Errorf("%s. Retrying in %d seconds.",
IrcCannotJoinError{ircChan, reason}.Error(),
ircOutput.TimeBeforeRejoin)
args := []string{"", ircChan, reason}
event := irc.Event{Code: IRC_ERR_CHANNELISFULL, Arguments: args}
c.Specify("logs an error after using up its attempts", func() {
mockIrcConn.failJoin = true
// We should leave the channel so that we aren't just
// causing a failure to join even when we're already in the
// channel
ircOutput.Conn.Part(ircChan)
maxRetries := int(ircOutput.MaxJoinRetries)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)
gomock.InOrder(
outTestHelper.MockOutputRunner.EXPECT().LogError(
err).Times(maxRetries),
outTestHelper.MockOutputRunner.EXPECT().LogError(
fmt.Errorf(ErrUsedUpRetryAttempts, ircChan)),
outTestHelper.MockOutputRunner.EXPECT().LogError(
ErrNoJoinableChannels),
)
for i := 1; i <= maxRetries; i++ {
// Trigger the event the number of times
ircOutput.Conn.RunCallbacks(&event)
c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, i)
}
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)
ircOutput.Conn.RunCallbacks(&event)
// We've used up our attempts,
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
ircOutput.Conn.RunCallbacks(&event)
c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
})
c.Specify("resets its attempts when it successful", func() {
mockIrcConn.failJoin = true
outTestHelper.MockOutputRunner.EXPECT().LogError(err).Times(2)
ircOutput.Conn.RunCallbacks(&event)
c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, 1)
mockIrcConn.failJoin = false
ircOutput.Conn.RunCallbacks(&event)
c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, 0)
})
close(inChan)
wg.Wait()
})
c.Specify("when banned from a server logs an error and quits", func() {
err := ircOutput.Init(config)
c.Assume(err, gs.IsNil)
startOutput()
event := irc.Event{Code: IRC_ERR_YOUREBANNEDCREEP}
outTestHelper.MockOutputRunner.EXPECT().LogError(ErrBannedFromServer)
ircOutput.Conn.RunCallbacks(&event)
// We should be able to exit without closing the channel first
wg.Wait()
// Clean this up
close(inChan)
})
c.Expect(<-errChan, gs.IsNil)
c.Expect(<-mockIrcConn.connected, gs.IsFalse)
})
// Cleanup which should happen each run
close(mockIrcConn.connected)
close(mockIrcConn.delivered)
close(mockIrcConn.Error)
})
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct())
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("w/ ProtobufEncoder", func() {
encoder := new(ProtobufEncoder)
encoder.Init(nil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
c.Specify("uses framing", func() {
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
})
c.Specify("but not if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.file.Close()
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
if runtime.GOOS != "windows" {
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = filepath.Join(tmpdir, "out.txt")
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
os.RemoveAll(tmpdir)
})
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
subdir := filepath.Join(os.TempDir(), "subdir")
config.Path = filepath.Join(subdir, "out.txt")
err := fileOutput.Init(config)
defer os.RemoveAll(subdir)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(subdir)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
}
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
runtime.Gosched() // Yield so we can overwrite the timerChan.
fileOutput.timerChan = timerChan
}
cleanUp := func() {
close(inChan)
fileOutput.file.Close()
wg.Done()
}
c.Specify("honors flush interval", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case _ = <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
out, err := encoder.Encode(pack2)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups.
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel.
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return.
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
bytesChan := make(chan []byte, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init.
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Set up relevant mocks.
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.MockSplitterRunner = NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a text message", func() {
// Create a channel to send data to the input. Drop a message on
// there and close the channel.
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "This is a message")
close(streamChan)
err = <-errChan
})
c.Specify("consumes a protobuf encoded message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, string(msgBody))
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
// Expect the close and the InChan calls.
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func TcpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests-")
os.MkdirAll(tmpDir, 0777)
defer func() {
ctrl.Finish()
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
pConfig := NewPipelineConfig(nil)
c.Specify("TcpOutput Internals", func() {
tcpOutput := new(TcpOutput)
tcpOutput.connection = pipeline_ts.NewMockConn(ctrl)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("SetName", func() {
tcpOutput.SetName("this-is a test")
c.Expect(tcpOutput.name, gs.Equals, "this_is_a_test")
})
c.Specify("fileExists", func() {
c.Expect(fileExists(tmpDir), gs.IsTrue)
c.Expect(fileExists(filepath.Join(tmpDir, "test.log")), gs.IsFalse)
})
c.Specify("extractBufferId", func() {
id, err := extractBufferId("555.log")
c.Expect(err, gs.IsNil)
c.Expect(id, gs.Equals, uint(555))
id, err = extractBufferId("")
c.Expect(err, gs.Not(gs.IsNil))
id, err = extractBufferId("a.log")
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("findBufferId", func() {
c.Expect(findBufferId(tmpDir, true), gs.Equals, uint(0))
c.Expect(findBufferId(tmpDir, false), gs.Equals, uint(0))
_, err := os.OpenFile(filepath.Join(tmpDir, "4.log"), os.O_CREATE, 0644)
c.Expect(err, gs.IsNil)
_, err = os.OpenFile(filepath.Join(tmpDir, "5.log"), os.O_CREATE, 0644)
c.Expect(err, gs.IsNil)
_, err = os.OpenFile(filepath.Join(tmpDir, "6a.log"), os.O_CREATE, 0644)
c.Expect(err, gs.IsNil)
c.Expect(findBufferId(tmpDir, false), gs.Equals, uint(4))
c.Expect(findBufferId(tmpDir, true), gs.Equals, uint(5))
})
c.Specify("writeCheckpoint", func() {
tcpOutput.checkpointFilename = filepath.Join(tmpDir, "cp.txt")
err := tcpOutput.writeCheckpoint(43, 99999)
c.Expect(err, gs.IsNil)
c.Expect(fileExists(tcpOutput.checkpointFilename), gs.IsTrue)
id, offset, err := readCheckpoint(tcpOutput.checkpointFilename)
c.Expect(err, gs.IsNil)
c.Expect(id, gs.Equals, uint(43))
c.Expect(offset, gs.Equals, int64(99999))
err = tcpOutput.writeCheckpoint(43, 1)
c.Expect(err, gs.IsNil)
id, offset, err = readCheckpoint(tcpOutput.checkpointFilename)
c.Expect(err, gs.IsNil)
c.Expect(id, gs.Equals, uint(43))
c.Expect(offset, gs.Equals, int64(1))
tcpOutput.checkpointFile.Close()
})
c.Specify("readCheckpoint", func() {
cp := filepath.Join(tmpDir, "cp.txt")
file, err := os.OpenFile(cp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
c.Expect(err, gs.IsNil)
id, offset, err := readCheckpoint(cp)
c.Expect(err, gs.Not(gs.IsNil))
file.WriteString("22")
id, offset, err = readCheckpoint(cp)
c.Expect(err.Error(), gs.Equals, "invalid checkpoint format")
file.Seek(0, 0)
file.WriteString("aa 22")
id, offset, err = readCheckpoint(cp)
c.Expect(err.Error(), gs.Equals, "invalid checkpoint id")
file.Seek(0, 0)
file.WriteString("43 aa")
id, offset, err = readCheckpoint(cp)
c.Expect(err.Error(), gs.Equals, "invalid checkpoint offset")
file.Seek(0, 0)
file.WriteString("43 22")
file.Close()
id, offset, err = readCheckpoint(cp)
c.Expect(err, gs.IsNil)
c.Expect(id, gs.Equals, uint(43))
c.Expect(offset, gs.Equals, int64(22))
})
c.Specify("writeToNextFile", func() {
tcpOutput.checkpointFilename = filepath.Join(tmpDir, "cp.txt")
tcpOutput.queue = tmpDir
err := tcpOutput.writeToNextFile()
c.Expect(err, gs.IsNil)
c.Expect(fileExists(getQueueFilename(tcpOutput.queue, tcpOutput.writeId)), gs.IsTrue)
tcpOutput.writeFile.WriteString("this is a test item")
tcpOutput.writeFile.Close()
tcpOutput.writeCheckpoint(tcpOutput.writeId, 10)
tcpOutput.checkpointFile.Close()
err = tcpOutput.readFromNextFile()
buf := make([]byte, 4)
n, err := tcpOutput.readFile.Read(buf)
c.Expect(n, gs.Equals, 4)
c.Expect("test", gs.Equals, string(buf))
tcpOutput.writeFile.Close()
tcpOutput.readFile.Close()
})
})
c.Specify("TcpOutput", func() {
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
}()
tcpOutput := new(TcpOutput)
tcpOutput.SetName("test")
config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
tcpOutput.Init(config)
tickChan := make(chan time.Time)
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("writes out to the network", func() {
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch // wait for server
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
go func() {
wg.Add(1)
c.Expect(err, gs.IsNil)
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
output := false
for x := 0; x < 5; x++ {
if fileExists(tcpOutput.checkpointFilename) {
output = true
break
}
time.Sleep(time.Duration(500) * time.Millisecond)
}
c.Expect(output, gs.Equals, true)
close(inChan)
wg.Wait() // wait for close to finish, prevents intermittent test failures
matchBytes := make([]byte, 0, 1000)
newpack := NewPipelinePack(nil)
newpack.Message = msg
newpack.Decoded = true
newpack.Message.SetPayload(outStr)
err = ProtobufEncodeMessage(newpack, &matchBytes)
c.Expect(err, gs.IsNil)
result = <-ch
c.Expect(result, gs.Equals, string(matchBytes))
id, offset, err := readCheckpoint(tcpOutput.checkpointFilename)
c.Expect(err, gs.IsNil)
c.Expect(id, gs.Equals, uint(1))
c.Expect(offset, gs.Equals, int64(115))
})
c.Specify("far end not initially listening", func() {
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
go func() {
wg.Add(1)
c.Expect(err, gs.IsNil)
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
output := false
for x := 0; x < 5; x++ {
if fileExists(getQueueFilename(tcpOutput.queue, tcpOutput.writeId)) {
output = true
time.Sleep(time.Duration(1) * time.Second) // give the send time to fail
break
}
time.Sleep(time.Duration(500) * time.Millisecond)
}
c.Expect(output, gs.Equals, true)
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch // wait for server
close(inChan)
wg.Wait() // wait for close to finish, prevents intermittent test failures
matchBytes := make([]byte, 0, 1000)
newpack := NewPipelinePack(nil)
newpack.Message = msg
newpack.Decoded = true
newpack.Message.SetPayload(outStr)
err = ProtobufEncodeMessage(newpack, &matchBytes)
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, string(matchBytes))
id, offset, err := readCheckpoint(tcpOutput.checkpointFilename)
c.Expect(err, gs.IsNil)
c.Expect(id, gs.Equals, uint(1))
c.Expect(offset, gs.Equals, int64(115))
})
})
}
func UnixOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
c.Specify("A UnixOutput", func() {
unixOutput := new(UdpOutput)
config := unixOutput.ConfigStruct().(*UdpOutputConfig)
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(new(plugins.PayloadEncoderConfig))
inChan := make(chan *pipeline.PipelinePack, 1)
rChan := make(chan *pipeline.PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
payload := "Write me out to the network."
msg.SetPayload(payload)
pack := pipeline.NewPipelinePack(rChan)
pack.Message = msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
c.Specify("using UDP", func() {
addr := "@/heka/unix-out"
config.Address = addr
ch := make(chan string, 1)
var wg sync.WaitGroup
var rAddr net.Addr
collectData := func() {
conn, err := net.ListenPacket("unix", addr)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := unixOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = "@/heka/unix-out"
err := unixOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "unix")
c.Expect(rAddr.String(), gs.Equals, "@/heka/unix-out")
close(inChan)
wg.Wait()
})
})
})
}
func NsqOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
var wg sync.WaitGroup
errChan := make(chan error, 1)
defer close(errChan)
c.Specify("A nsq output", func() {
output := new(NsqOutput)
oth := plugins_ts.NewOutputTestHelper(ctrl)
config := output.ConfigStruct().(*NsqOutputConfig)
config.Topic = "test"
startOutput := func() {
wg.Add(1)
go func() {
errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
}
var mockProducer *MockProducer
output.newProducer = func(addr string, config *nsq.Config) (Producer, error) {
mockProducer, _ = NewMockProducer(addr, config)
return mockProducer, nil
}
msg := pipeline_ts.GetTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
c.Specify("requires at least one address", func() {
err := output.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
config.Addresses = []string{"test.com:4150"}
c.Specify("requires an encoder", func() {
err := output.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(nil)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
encoder := new(plugins.PayloadEncoder)
encoder.Init(&plugins.PayloadEncoderConfig{PrefixTs: false})
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
inChan := make(chan *pipeline.PipelinePack, 1)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
c.Specify("publishes a message with the configured topic", func() {
err := output.Init(config)
c.Assume(err, gs.IsNil)
startOutput()
inChan <- pack
transaction := <-mockProducer.respChan
c.Expect(transaction.Error, gs.IsNil)
close(inChan)
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
})
})
})
}
func OutputSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := pipeline.NewPipelineConfig(nil)
inChan := make(chan *pipeline.PipelinePack, 1)
timer := make(chan time.Time, 1)
c.Specify("A SandboxOutput", func() {
output := new(SandboxOutput)
output.SetPipelineConfig(pConfig)
conf := output.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/output.lua"
conf.ModuleDirectory = "../lua/modules;../lua/testsupport/modules"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
data := "1376389920 debug id=2321 url=example.com item=1"
pack.Message.SetPayload(data)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Ticker().Return(timer)
c.Specify("writes a payload to file", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
inChan <- pack
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Assume(err, gs.IsNil)
tmpFile, err := os.Open("output.lua.txt")
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, data)
c.Expect(output.processMessageCount, gs.Equals, int64(1))
c.Expect(output.processMessageSamples, gs.Equals, int64(1))
c.Expect(output.processMessageFailures, gs.Equals, int64(0))
})
c.Specify("failure processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
pack.Message.SetPayload("FAILURE")
pack.BufferedPack = true
pack.DelivErrChan = make(chan error, 1)
inChan <- pack
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Assume(err, gs.IsNil)
c.Expect(output.processMessageFailures, gs.Equals, int64(1))
err = <-pack.DelivErrChan
c.Expect(err.Error(), gs.Equals, "failure message")
})
c.Specify("user abort processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
e := errors.New("FATAL: user abort")
pack.Message.SetPayload("USERABORT")
inChan <- pack
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err.Error(), gs.Equals, e.Error())
})
c.Specify("fatal error processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
pack.Message.SetPayload("FATAL")
inChan <- pack
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("fatal error in timer_event", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
timer <- time.Now()
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("fatal error in shutdown timer_event", func() {
conf.TimerEventOnShutdown = true
err := output.Init(conf)
c.Expect(err, gs.IsNil)
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func TestSendMessage(t *testing.T) {
ctrl := gomock.NewController(t)
broker := sarama.NewMockBroker(t, 2)
defer func() {
broker.Close()
ctrl.Finish()
}()
topic := "test"
globals := DefaultGlobals()
pConfig := NewPipelineConfig(globals)
broker.SetHandlerByMap(map[string]sarama.MockResponse{
"MetadataRequest": sarama.NewMockMetadataResponse(t).
SetBroker(broker.Addr(), broker.BrokerID()).
SetLeader(topic, 0, broker.BrokerID()),
"ProduceRequest": sarama.NewMockProduceResponse(t),
})
ko := new(KafkaOutput)
ko.SetPipelineConfig(pConfig)
config := ko.ConfigStruct().(*KafkaOutputConfig)
config.Addrs = append(config.Addrs, broker.Addr())
config.Topic = topic
err := ko.Init(config)
if err != nil {
t.Fatal(err)
}
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct().(*plugins.PayloadEncoderConfig))
inChan := make(chan *PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
errChan := make(chan error)
startOutput := func() {
go func() {
err := ko.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
}
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
startOutput()
msgcount := atomic.LoadInt64(&ko.processMessageCount)
if msgcount != 0 {
t.Errorf("Invalid starting processMessageCount %d", msgcount)
}
msgcount = atomic.LoadInt64(&ko.processMessageFailures)
if msgcount != 0 {
t.Errorf("Invalid starting processMessageFailures %d", msgcount)
}
inChan <- pack
close(inChan)
err = <-errChan
if err != nil {
t.Errorf("Error running output %s", err)
}
msgcount = atomic.LoadInt64(&ko.processMessageCount)
if msgcount != 1 {
t.Errorf("Invalid ending processMessageCount %d", msgcount)
}
msgcount = atomic.LoadInt64(&ko.processMessageFailures)
if msgcount != 0 {
t.Errorf("Invalid ending processMessageFailures %d", msgcount)
}
}
func TcpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests")
defer func() {
ctrl.Finish()
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
pConfig := NewPipelineConfig(globals)
pConfig.RegisterDefault("HekaFramingSplitter")
c.Specify("TcpOutput", func() {
tcpOutput := new(TcpOutput)
tcpOutput.SetName("test")
config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
tcpOutput.Init(config)
tickChan := make(chan time.Time)
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan).AnyTimes()
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.QueueCursor = "queuecursor"
outStr := "Write me out to the network"
newpack := NewPipelinePack(nil)
newpack.Message = msg
newpack.Message.SetPayload(outStr)
matchBytes, err := proto.Marshal(newpack.Message)
c.Assume(err, gs.IsNil)
pack.MsgBytes = matchBytes
newpack.MsgBytes = matchBytes
oth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
c.Specify("doesn't use framing w/o ProtobufEncoder", func() {
encoder := new(plugins.PayloadEncoder)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
c.Specify("doesn't use framing if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
c.Specify("writes out to the network", func() {
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch // wait for server
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)
pack.Message.SetPayload(outStr)
msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
err = tcpOutput.ProcessMessage(pack)
c.Expect(err, gs.IsNil)
result = <-ch
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(1))
c.Expect(result, gs.Equals, string(matchBytes))
tcpOutput.CleanUp()
})
c.Specify("far end not initially listening", func() {
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper)
c.Assume(err, gs.IsNil)
pack.Message.SetPayload(outStr)
msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
err = tcpOutput.ProcessMessage(pack)
_, ok := err.(RetryMessageError)
c.Expect(ok, gs.IsTrue)
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
// After the message is queued start the collector. However, we
// don't have a way guarantee a send attempt has already been made
// and that we are actually exercising the retry code.
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)
err = tcpOutput.ProcessMessage(pack)
c.Expect(err, gs.IsNil)
result = <-ch
c.Expect(result, gs.Equals, string(matchBytes))
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(1))
tcpOutput.CleanUp()
})
// c.Specify("Overload queue drops messages", func() {
// config.QueueFullAction = "drop"
// config.QueueMaxBufferSize = uint64(1)
// use_framing := false
// config.UseFraming = &use_framing
// oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
// oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)
// err := tcpOutput.Init(config)
// c.Expect(err, gs.IsNil)
// startOutput()
// inChan <- pack
// dropcount := atomic.LoadInt64(&tcpOutput.dropMessageCount)
// for x := 0; x < 5 && dropcount == 0; x++ {
// dropcount = atomic.LoadInt64(&tcpOutput.dropMessageCount)
// time.Sleep(time.Duration(100) * time.Millisecond)
// }
// c.Expect(dropcount, gs.Equals, int64(1))
// close(inChan)
// })
// c.Specify("Overload queue shutdowns Heka", func() {
// config.QueueFullAction = "shutdown"
// config.QueueMaxBufferSize = uint64(1)
// use_framing := false
// config.UseFraming = &use_framing
// oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
// oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)
// sigChan := globals.SigChan()
// err := tcpOutput.Init(config)
// c.Expect(err, gs.IsNil)
// startOutput()
// inChan <- pack
// shutdownSignal := <-sigChan
// c.Expect(shutdownSignal, gs.Equals, syscall.SIGINT)
// close(inChan)
// })
// c.Specify("Overload queue blocks processing until packet is sent", func() {
// config.QueueFullAction = "block"
// config.QueueMaxBufferSize = uint64(1)
// use_framing := false
// config.UseFraming = &use_framing
// oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes()
// oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)
// err := tcpOutput.Init(config)
// c.Expect(err, gs.IsNil)
// startOutput()
// inChan <- pack
// msgcount := atomic.LoadInt64(&tcpOutput.dropMessageCount)
// for x := 0; x < 5 && msgcount == 0; x++ {
// msgcount = atomic.LoadInt64(&tcpOutput.dropMessageCount)
// time.Sleep(time.Duration(100) * time.Millisecond)
// }
// c.Expect(atomic.LoadInt64(&tcpOutput.dropMessageCount), gs.Equals, int64(0))
// c.Expect(atomic.LoadInt64(&tcpOutput.processMessageCount), gs.Equals, int64(0))
// close(inChan)
// })
})
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
tmpFileName)
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
toString := func(outData interface{}) string {
return string(*(outData.(*[]byte)))
}
c.Specify("correctly formats text output", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 20)
c.Specify("by default", func() {
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
})
c.Specify("w/ a prepended timestamp when specified", func() {
fileOutput.prefix_ts = true
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
payload := *msg.Payload
c.Expect(strContents, pipeline_ts.StringContains, payload)
c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
})
c.Specify("even when payload is nil", func() {
pack.Message.Payload = nil
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
strContents := toString(&outData)
c.Expect(strContents, gs.Equals, "\n")
})
c.Specify("payload is nil and with a timestamp", func() {
pack.Message.Payload = nil
fileOutput.prefix_ts = true
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
c.Expect(strings.HasPrefix(strContents, todayStr), gs.IsTrue)
c.Expect(strings.HasSuffix(strContents, " \n"), gs.IsTrue)
})
})
c.Specify("correctly formats JSON output", func() {
config.Format = "json"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified", func() {
fileOutput.handleMessage(pack, &outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
})
c.Specify("and with a timestamp", func() {
fileOutput.prefix_ts = true
fileOutput.handleMessage(pack, &outData)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(strContents, pipeline_ts.StringContains, string(msgJson)+"\n")
c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
})
})
c.Specify("correctly formats protocol buffer stream output", func() {
config.Format = "protobufstream"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified and timestamp ignored", func() {
fileOutput.prefix_ts = true
err := fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = tmpdir
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
config.Path = filepath.Join(tmpFilePath, "subfile")
err := fileOutput.Init(config)
defer os.Remove(config.Path)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(tmpFilePath)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
runtime.Gosched() // Yield so we can overwrite the timerChan.
fileOutput.timerChan = timerChan
}
cleanUp := func() {
close(inChan)
wg.Done()
}
c.Specify("honors flush interval", func() {
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case _ = <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}
func HttpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
var (
reqMethod string
reqBody string
reqHeader http.Header
handleWg sync.WaitGroup
runWg sync.WaitGroup
delay bool
)
handler := new(_test_handler)
handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) {
defer handleWg.Done()
if delay {
time.Sleep(time.Duration(2) * time.Millisecond)
}
// Capture request body for test assertions.
p := make([]byte, req.ContentLength)
_, err := req.Body.Read(p)
c.Expect(err.Error(), gs.Equals, "EOF")
reqBody = string(p)
// Capture request method and headers for test assertions.
reqMethod = req.Method
reqHeader = req.Header
// Respond with either a response body or a response code.
if len(handler.respBody) > 0 {
rw.Write([]byte(handler.respBody))
} else {
rw.WriteHeader(handler.respCode)
}
flusher := rw.(http.Flusher)
flusher.Flush()
}
oth := ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encConfig := new(plugins.PayloadEncoderConfig)
err := encoder.Init(encConfig)
c.Expect(err, gs.IsNil)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
c.Specify("An HttpOutput", func() {
httpOutput := new(HttpOutput)
config := httpOutput.ConfigStruct().(*HttpOutputConfig)
c.Specify("barfs on bogus URLs", func() {
config.Address = "one-two-three-four"
err := httpOutput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
server := httptest.NewServer(handler)
defer server.Close()
runOutput := func() {
httpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
runWg.Done()
}
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
payload := "this is the payload"
pack.Message.SetPayload(payload)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(
[]byte(payload), nil)
config.Address = server.URL
handler.respBody = "Response Body"
c.Specify("makes http POST requests by default", func() {
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(reqMethod, gs.Equals, "POST")
})
c.Specify("makes http PUT requests", func() {
config.Method = "put"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(reqMethod, gs.Equals, "PUT")
})
c.Specify("makes http GET requests", func() {
config.Method = "get"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, "")
c.Expect(reqMethod, gs.Equals, "GET")
})
c.Specify("correctly passes headers along", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"]))
c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"]))
})
c.Specify("uses http auth when specified", func() {
config.Username = "******"
config.Password = "******"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
auth := reqHeader.Get("Authorization")
c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue)
decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:])
c.Expect(err, gs.IsNil)
c.Expect(string(decodedAuth), gs.Equals, "user:pass")
})
c.Specify("logs error responses", func() {
handler.respBody = ""
handler.respCode = 500
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
var errMsg string
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
func(err error) {
errMsg = err.Error()
})
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(strings.HasPrefix(errMsg,
"HTTP Error code returned: 500"), gs.IsTrue)
})
c.Specify("honors http timeout interval", func() {
config.HttpTimeout = 1 // 1 millisecond
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
var errMsg string
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
func(err error) {
errMsg = err.Error()
})
delay = true
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(strings.Contains(errMsg, "use of closed network connection"),
gs.IsTrue)
})
})
})
}
func SmtpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
c.Specify("A SmtpOutput", func() {
smtpOutput := new(SmtpOutput)
config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
config.SendTo = []string{"root"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
runnerName.Return("SmtpOutput")
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
encCall := oth.MockOutputRunner.EXPECT().Encode(pack)
c.Specify("send email payload message", func() {
err := smtpOutput.Init(config)
c.Assume(err, gs.IsNil)
smtpOutput.sendFunction = testSendMail
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
encCall.Return(encoder.Encode(pack))
wg.Add(1)
go func() {
smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait()
})
})
c.Specify("SmtpOutput Message Body Encoding", func() {
smtpOutput := new(SmtpOutput)
chars := "123456789012345678901234567890123456789012345678901234567"
charsE := "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3"
examples := [][]string{
{"Hello", "SGVsbG8="},
{chars, charsE},
{chars + chars, charsE + "\r\n" + charsE},
{chars + chars + "Hello", charsE + "\r\n" + charsE + "\r\n" + "SGVsbG8="},
{"", ""},
{"1", "MQ=="},
}
for _, example := range examples {
smtpOutput.encodeFullMsg([]byte(example[0]))
c.Expect(string(smtpOutput.fullMsg), gs.Equals, example[1])
}
})
// // Use this test with a real server
// c.Specify("Real SmtpOutput output", func() {
// smtpOutput := new(SmtpOutput)
// config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
// config.SendTo = []string{"root"}
// msg := pipeline_ts.GetTestMessage()
// pack := NewPipelinePack(pConfig.InputRecycleChan())
// pack.Message = msg
// pack.Decoded = true
// inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
// inChanCall.Return(inChan)
// runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
// runnerName.Return("SmtpOutput")
// oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
// encCall := oth.MockOutputRunner.EXPECT().Encode(pack)
// c.Specify("send a real email essage", func() {
// err := smtpOutput.Init(config)
// c.Assume(err, gs.IsNil)
// outStr := "Write me out to the network"
// pack.Message.SetPayload(outStr)
// encCall.Return(encoder.Encode(pack))
// go func() {
// wg.Add(1)
// smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
// wg.Done()
// }()
// inChan <- pack
// time.Sleep(1000) // allow time for the message output
// close(inChan)
// wg.Wait()
// // manually check the mail
// })
// })
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct())
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.QueueCursor = "queuecursor"
errChan := make(chan error, 1)
c.Specify("w/ ProtobufEncoder", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
c.Specify("uses framing", func() {
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
})
c.Specify("but not if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
})
c.Specify("rotates files correctly", func() {
config.Path = "%Y-%m-%d"
config.RotationInterval = 24
rotateChan := make(chan time.Time)
closingChan := make(chan struct{})
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
defer fileOutput.file.Close()
fileOutput.rotateChan = rotateChan
fileOutput.closing = closingChan
fileOutput.startRotateNotifier()
committerChan := make(chan struct{})
go func() {
fileOutput.committer(oth.MockOutputRunner, errChan)
close(committerChan)
}()
c.Assume(fileOutput.path, gs.Equals, time.Now().Format("2006-01-02"))
futureDuration, _ := time.ParseDuration("24h")
futureNow := time.Now().Add(futureDuration)
rotateChan <- futureNow
close(inChan)
close(fileOutput.batchChan)
<-committerChan
c.Assume(fileOutput.path, gs.Equals, futureNow.Format("2006-01-02"))
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.file.Close()
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
go fileOutput.receiver(oth.MockOutputRunner, errChan)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
c.Expect(string(outBatch.data), gs.Equals, payload)
c.Expect(outBatch.cursor, gs.Equals, pack.QueueCursor)
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
batch := &outBatch{
data: outBytes,
cursor: pack.QueueCursor,
}
oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop.
go fileOutput.committer(oth.MockOutputRunner, errChan)
// Feed and close the batchChan.
go func() {
fileOutput.batchChan <- batch
_ = <-fileOutput.backChan // clear backChan to prevent blocking.
close(fileOutput.batchChan)
}()
// Wait until we know processing has finished.
<-fileOutput.closing
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop.
go fileOutput.committer(oth.MockOutputRunner, errChan)
// Feed and close the batchChan.
go func() {
fileOutput.batchChan <- batch
_ = <-fileOutput.backChan // clear backChan to prevent blocking.
close(fileOutput.batchChan)
}()
// Wait until we know processing has finished.
<-fileOutput.closing
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other.
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
if runtime.GOOS != "windows" {
if u, err := user.Current(); err != nil && u.Uid != "0" {
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = filepath.Join(tmpdir, "out.txt")
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
os.RemoveAll(tmpdir)
})
}
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
subdir := filepath.Join(os.TempDir(), "subdir")
config.Path = filepath.Join(subdir, "out.txt")
err := fileOutput.Init(config)
defer os.RemoveAll(subdir)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(subdir)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
}
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
msg2.SetPayload("MESSAGE 2")
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.timerChan = timerChan
go fileOutput.receiver(oth.MockOutputRunner, errChan)
runtime.Gosched() // Yield so receiver will start.
}
cleanUp := func() {
close(inChan)
fileOutput.file.Close()
}
c.Specify("honors flush interval", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case _ = <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
after = time.After(100 * time.Millisecond)
inChan <- pack2
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
out, err := encoder.Encode(pack2)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
inChan <- pack2
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}
