func SmtpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
c.Specify("A SmtpOutput", func() {
smtpOutput := new(SmtpOutput)
config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
config.SendTo = []string{"root"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
runnerName.Return("SmtpOutput")
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()
c.Specify("send email payload message", func() {
err := smtpOutput.Init(config)
c.Assume(err, gs.IsNil)
smtpOutput.sendFunction = testSendMail
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
go func() {
wg.Add(1)
smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait()
})
})
// Use this test with a real server
// c.Specify("Real SmtpOutput output", func() {
// smtpOutput := new(SmtpOutput)
//
// config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
// config.SendTo = []string{"root"}
//
// msg := pipeline_ts.GetTestMessage()
// pack := NewPipelinePack(pConfig.InputRecycleChan())
// pack.Message = msg
// pack.Decoded = true
// inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
// inChanCall.Return(inChan)
// runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
// runnerName.Return("SmtpOutput")
// oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()
//
// c.Specify("send a real email essage", func() {
//
// err := smtpOutput.Init(config)
// c.Assume(err, gs.IsNil)
//
// outStr := "Write me out to the network"
// pack.Message.SetPayload(outStr)
// go func() {
// wg.Add(1)
// smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
// wg.Done()
// }()
// inChan <- pack
// time.Sleep(1000) // allow time for the message output
// close(inChan)
// wg.Wait()
// // manually check the mail
// })
// })
}
func SmtpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
c.Specify("A SmtpOutput", func() {
smtpOutput := new(SmtpOutput)
config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
config.SendTo = []string{"root"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
runnerName.Return("SmtpOutput")
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
encCall := oth.MockOutputRunner.EXPECT().Encode(pack)
c.Specify("send email payload message", func() {
err := smtpOutput.Init(config)
c.Assume(err, gs.IsNil)
smtpOutput.sendFunction = testSendMail
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
encCall.Return(encoder.Encode(pack))
wg.Add(1)
go func() {
smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait()
})
})
c.Specify("SmtpOutput Message Body Encoding", func() {
smtpOutput := new(SmtpOutput)
chars := "123456789012345678901234567890123456789012345678901234567"
charsE := "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3"
examples := [][]string{
{"Hello", "SGVsbG8="},
{chars, charsE},
{chars + chars, charsE + "\r\n" + charsE},
{chars + chars + "Hello", charsE + "\r\n" + charsE + "\r\n" + "SGVsbG8="},
{"", ""},
{"1", "MQ=="},
}
for _, example := range examples {
smtpOutput.encodeFullMsg([]byte(example[0]))
c.Expect(string(smtpOutput.fullMsg), gs.Equals, example[1])
}
})
// // Use this test with a real server
// c.Specify("Real SmtpOutput output", func() {
// smtpOutput := new(SmtpOutput)
// config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
// config.SendTo = []string{"root"}
// msg := pipeline_ts.GetTestMessage()
// pack := NewPipelinePack(pConfig.InputRecycleChan())
// pack.Message = msg
// pack.Decoded = true
// inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
// inChanCall.Return(inChan)
// runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
// runnerName.Return("SmtpOutput")
// oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
// encCall := oth.MockOutputRunner.EXPECT().Encode(pack)
// c.Specify("send a real email essage", func() {
// err := smtpOutput.Init(config)
// c.Assume(err, gs.IsNil)
// outStr := "Write me out to the network"
// pack.Message.SetPayload(outStr)
// encCall.Return(encoder.Encode(pack))
// go func() {
// wg.Add(1)
// smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
// wg.Done()
// }()
// inChan <- pack
// time.Sleep(1000) // allow time for the message output
// close(inChan)
// wg.Wait()
// // manually check the mail
// })
// })
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockDRunner := NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
c.Specify("with a valid setup and no decoder", func() {
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a message", func() {
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
})
c.Specify("with a valid setup using a decoder", func() {
decoderName := "defaultDecoder"
config.Decoder = decoderName
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
// Mock up our default decoder runner and decoder.
ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
decCall.Return(mockDRunner, true)
mockDecoder := NewMockDecoder(ctrl)
mockDRunner.EXPECT().Decoder().Return(mockDecoder)
c.Specify("consumes a message", func() {
packs := []*PipelinePack{ith.Pack}
mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
c.Specify("consumes a serialized message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the decoded pack
mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
// Ignore leading 5 bytes of encoded message as thats the header
c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close
ug.Add(1)
// Expect the close and the InChan calls
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct())
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("w/ ProtobufEncoder", func() {
encoder := new(ProtobufEncoder)
encoder.Init(nil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
c.Specify("uses framing", func() {
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
})
c.Specify("but not if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.file.Close()
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
if runtime.GOOS != "windows" {
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = filepath.Join(tmpdir, "out.txt")
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
os.RemoveAll(tmpdir)
})
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
subdir := filepath.Join(os.TempDir(), "subdir")
config.Path = filepath.Join(subdir, "out.txt")
err := fileOutput.Init(config)
defer os.RemoveAll(subdir)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(subdir)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
}
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
runtime.Gosched() // Yield so we can overwrite the timerChan.
fileOutput.timerChan = timerChan
}
cleanUp := func() {
close(inChan)
fileOutput.file.Close()
wg.Done()
}
c.Specify("honors flush interval", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case _ = <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
out, err := encoder.Encode(pack2)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups.
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel.
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return.
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
bytesChan := make(chan []byte, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init.
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Set up relevant mocks.
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.MockSplitterRunner = NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a text message", func() {
// Create a channel to send data to the input. Drop a message on
// there and close the channel.
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "This is a message")
close(streamChan)
err = <-errChan
})
c.Specify("consumes a protobuf encoded message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, string(msgBody))
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
// Expect the close and the InChan calls.
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func TestSendMessage(t *testing.T) {
ctrl := gomock.NewController(t)
broker := sarama.NewMockBroker(t, 2)
defer func() {
broker.Close()
ctrl.Finish()
}()
topic := "test"
globals := DefaultGlobals()
pConfig := NewPipelineConfig(globals)
broker.SetHandlerByMap(map[string]sarama.MockResponse{
"MetadataRequest": sarama.NewMockMetadataResponse(t).
SetBroker(broker.Addr(), broker.BrokerID()).
SetLeader(topic, 0, broker.BrokerID()),
"ProduceRequest": sarama.NewMockProduceResponse(t),
})
ko := new(KafkaOutput)
ko.SetPipelineConfig(pConfig)
config := ko.ConfigStruct().(*KafkaOutputConfig)
config.Addrs = append(config.Addrs, broker.Addr())
config.Topic = topic
err := ko.Init(config)
if err != nil {
t.Fatal(err)
}
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct().(*plugins.PayloadEncoderConfig))
inChan := make(chan *PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
errChan := make(chan error)
startOutput := func() {
go func() {
err := ko.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
}
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
startOutput()
msgcount := atomic.LoadInt64(&ko.processMessageCount)
if msgcount != 0 {
t.Errorf("Invalid starting processMessageCount %d", msgcount)
}
msgcount = atomic.LoadInt64(&ko.processMessageFailures)
if msgcount != 0 {
t.Errorf("Invalid starting processMessageFailures %d", msgcount)
}
inChan <- pack
close(inChan)
err = <-errChan
if err != nil {
t.Errorf("Error running output %s", err)
}
msgcount = atomic.LoadInt64(&ko.processMessageCount)
if msgcount != 1 {
t.Errorf("Invalid ending processMessageCount %d", msgcount)
}
msgcount = atomic.LoadInt64(&ko.processMessageFailures)
if msgcount != 0 {
t.Errorf("Invalid ending processMessageFailures %d", msgcount)
}
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct())
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.QueueCursor = "queuecursor"
errChan := make(chan error, 1)
c.Specify("w/ ProtobufEncoder", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
c.Specify("uses framing", func() {
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
})
c.Specify("but not if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
})
c.Specify("rotates files correctly", func() {
config.Path = "%Y-%m-%d"
config.RotationInterval = 24
rotateChan := make(chan time.Time)
closingChan := make(chan struct{})
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
defer fileOutput.file.Close()
fileOutput.rotateChan = rotateChan
fileOutput.closing = closingChan
fileOutput.startRotateNotifier()
committerChan := make(chan struct{})
go func() {
fileOutput.committer(oth.MockOutputRunner, errChan)
close(committerChan)
}()
c.Assume(fileOutput.path, gs.Equals, time.Now().Format("2006-01-02"))
futureDuration, _ := time.ParseDuration("24h")
futureNow := time.Now().Add(futureDuration)
rotateChan <- futureNow
close(inChan)
close(fileOutput.batchChan)
<-committerChan
c.Assume(fileOutput.path, gs.Equals, futureNow.Format("2006-01-02"))
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.file.Close()
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
go fileOutput.receiver(oth.MockOutputRunner, errChan)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
c.Expect(string(outBatch.data), gs.Equals, payload)
c.Expect(outBatch.cursor, gs.Equals, pack.QueueCursor)
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
batch := &outBatch{
data: outBytes,
cursor: pack.QueueCursor,
}
oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop.
go fileOutput.committer(oth.MockOutputRunner, errChan)
// Feed and close the batchChan.
go func() {
fileOutput.batchChan <- batch
_ = <-fileOutput.backChan // clear backChan to prevent blocking.
close(fileOutput.batchChan)
}()
// Wait until we know processing has finished.
<-fileOutput.closing
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop.
go fileOutput.committer(oth.MockOutputRunner, errChan)
// Feed and close the batchChan.
go func() {
fileOutput.batchChan <- batch
_ = <-fileOutput.backChan // clear backChan to prevent blocking.
close(fileOutput.batchChan)
}()
// Wait until we know processing has finished.
<-fileOutput.closing
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other.
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
if runtime.GOOS != "windows" {
if u, err := user.Current(); err != nil && u.Uid != "0" {
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = filepath.Join(tmpdir, "out.txt")
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
os.RemoveAll(tmpdir)
})
}
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
subdir := filepath.Join(os.TempDir(), "subdir")
config.Path = filepath.Join(subdir, "out.txt")
err := fileOutput.Init(config)
defer os.RemoveAll(subdir)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(subdir)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
}
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
msg2.SetPayload("MESSAGE 2")
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.timerChan = timerChan
go fileOutput.receiver(oth.MockOutputRunner, errChan)
runtime.Gosched() // Yield so receiver will start.
}
cleanUp := func() {
close(inChan)
fileOutput.file.Close()
}
c.Specify("honors flush interval", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case _ = <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
after = time.After(100 * time.Millisecond)
inChan <- pack2
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
after := time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
case <-after:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
out, err := encoder.Encode(pack2)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
inChan <- pack2
after = time.After(100 * time.Millisecond)
select {
case <-fileOutput.batchChan:
case <-after:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}