func getEmptyKeenOutputPack() *pipeline.PipelinePack {
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message.SetType("KeenOutput")
pack.Decoded = true
pack.Message.SetTimestamp(int64(EPOCH_TS * 1e9))
return pack
}
func getSentryPack() (pack *pipeline.PipelinePack) {
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack = pipeline.NewPipelinePack(recycleChan)
pack.Message.SetType("sentry")
fTimeStamp, _ := message.NewField("epoch_timestamp", EPOCH_TS, "utc-seconds")
fDsn, _ := message.NewField("dsn", DSN, "uri")
pack.Message.AddField(fTimeStamp)
pack.Message.AddField(fDsn)
pack.Message.SetPayload(PAYLOAD)
pack.Decoded = true
return
}
func getStatsdPack(typeStr string, payload string) (pack *pipeline.PipelinePack) {
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack = pipeline.NewPipelinePack(recycleChan)
pack.Message.SetType(typeStr)
pack.Message.SetLogger("thenamespace")
fName, _ := message.NewField("name", "myname", "")
fRate, _ := message.NewField("rate", .30, "")
pack.Message.AddField(fName)
pack.Message.AddField(fRate)
pack.Message.SetPayload(payload)
pack.Decoded = true
return pack
}
func RstEncoderSpec(c gs.Context) {
c.Specify("A RstEncoder", func() {
encoder := new(RstEncoder)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
payload := "This is the payload!"
pack.Message.SetPayload(payload)
loc, err := time.LoadLocation("America/Chicago")
c.Assume(err, gs.IsNil)
timestamp := time.Date(1971, 10, 7, 18, 47, 0, 0, loc)
pack.Message.SetTimestamp(timestamp.UnixNano())
pack.Message.SetType("test.type")
pack.Message.SetHostname("somehost.example.com")
pack.Message.SetPid(12345)
pack.Message.SetUuid(uuid.Parse("72de6a05-1b99-4a88-84c2-90797624c68f"))
pack.Message.SetLogger("loggyloglog")
pack.Message.SetEnvVersion("0.8")
field, err := message.NewField("intfield", 23, "count")
c.Assume(err, gs.IsNil)
field.AddValue(24)
field.AddValue(25)
pack.Message.AddField(field)
field, err = message.NewField("stringfield", "hold", "foreigner")
c.Assume(err, gs.IsNil)
field.AddValue("your")
field.AddValue("head")
field.AddValue("up")
pack.Message.AddField(field)
field, err = message.NewField("bool", true, "")
c.Assume(err, gs.IsNil)
field.AddValue(false)
pack.Message.AddField(field)
field, err = message.NewField("bool", false, "false")
c.Assume(err, gs.IsNil)
pack.Message.AddField(field)
field, err = message.NewField("float", 345.6789, "kelvin")
c.Assume(err, gs.IsNil)
field.AddValue(139847987987987.878732819)
pack.Message.AddField(field)
field, err = message.NewField("bytes", []byte("encode me"), "binary")
c.Assume(err, gs.IsNil)
field.AddValue([]byte("and me"))
field.AddValue([]byte("and me too"))
pack.Message.AddField(field)
expected := `
:Timestamp: 1971-10-07 23:47:00 +0000 UTC
:Type: test.type
:Hostname: somehost.example.com
:Pid: 12345
:Uuid: 72de6a05-1b99-4a88-84c2-90797624c68f
:Logger: loggyloglog
:Payload: This is the payload!
:EnvVersion: 0.8
:Severity: 7
:Fields:
| name:"intfield" type:integer value:[23,24,25] representation:"count"
| name:"stringfield" type:string value:["hold","your","head","up"] representation:"foreigner"
| name:"bool" type:bool value:[true,false]
| name:"bool" type:bool value:false representation:"false"
| name:"float" type:double value:[345.6789,1.3984798798798788e+14] representation:"kelvin"
| name:"bytes" type:bytes value:[ZW5jb2RlIG1l,YW5kIG1l,YW5kIG1lIHRvbw==] representation:"binary"
`
c.Specify("serializes a message correctly", func() {
err = encoder.Init(nil)
c.Assume(err, gs.IsNil)
output, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
c.Expect(string(output), gs.Equals, expected)
})
})
}
func UdpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
udpOutput := new(UdpOutput)
config := udpOutput.ConfigStruct().(*UdpOutputConfig)
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(new(plugins.PayloadEncoderConfig))
inChan := make(chan *pipeline.PipelinePack, 1)
rChan := make(chan *pipeline.PipelinePack, 1)
var wg sync.WaitGroup
var rAddr net.Addr
c.Specify("A UdpOutput", func() {
msg := pipeline_ts.GetTestMessage()
payload := "Write me out to the network."
msg.SetPayload(payload)
pack := pipeline.NewPipelinePack(rChan)
pack.Message = msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
c.Specify("using UDP", func() {
addr := "127.0.0.1:45678"
config.Address = addr
ch := make(chan string, 1)
collectData := func() {
conn, err := net.ListenPacket("udp", addr)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = "localhost:12345"
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "udp")
c.Expect(rAddr.String(), gs.Equals, "127.0.0.1:12345")
close(inChan)
wg.Wait()
})
})
c.Specify("using Unix datagrams", func() {
if runtime.GOOS == "windows" {
return
}
testUnixAddr := func() string {
f, err := ioutil.TempFile("", "_heka_test_sock")
c.Assume(err, gs.IsNil)
addr := f.Name()
f.Close()
os.Remove(addr)
return addr
}
config.Address = testUnixAddr()
config.Net = "unixgram"
ch := make(chan string, 1)
var wg sync.WaitGroup
var rAddr net.Addr
collectData := func() {
conn, err := net.ListenPacket("unixgram", config.Address)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
err = os.Remove(config.Address)
var errMsg string
if err != nil {
errMsg = err.Error()
}
ch <- errMsg
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
result = <-ch
c.Expect(result, gs.Equals, "")
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = testUnixAddr()
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "unixgram")
c.Expect(rAddr.String(), gs.Equals, config.LocalAddress)
close(inChan)
wg.Wait()
})
})
})
c.Specify("drop message contents if their size is bigger than allowed UDP datagram size", func() {
huge_msg := pipeline_ts.GetTestMessage()
payload := strings.Repeat("2", 131014)
huge_msg.SetPayload(payload)
huge_pack := pipeline.NewPipelinePack(rChan)
huge_pack.Message = huge_msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(huge_pack).Return(encoder.Encode(huge_pack))
oth.MockOutputRunner.EXPECT().LogError(fmt.Errorf("Message has exceeded allowed UDP data size: 131014 > 65507"))
config.Address = "localhost:12345"
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- huge_pack
close(inChan)
wg.Wait()
})
c.Specify("checks validation of of Maximum message size limit", func() {
config.Address = "localhost:12345"
config.MaxMessageSize = 100
err := udpOutput.Init(config)
c.Assume(err.Error(), gs.Equals, "Maximum message size can't be smaller than 512 bytes.")
})
}
func CloudwatchInputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
c.Specify("A CloudwatchInput", func() {
input := new(CloudwatchInput)
inputConfig := input.ConfigStruct().(*CloudwatchInputConfig)
inputConfig.MetricName = "Test"
inputConfig.Statistics = []string{"Average"}
inputConfig.PollInterval = "1ms"
inputConfig.Region = "us-east-1"
inputConfig.Namespace = "Testing"
err := input.Init(inputConfig)
c.Assume(err, gs.IsNil)
serv := ts.NewMockAWSService(ctrl)
input.cw.Service = serv
ith := new(InputTestHelper)
recycleChan := make(chan *pipeline.PipelinePack, 500)
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = ts.NewMockPluginHelper(ctrl)
ith.MockInputRunner = ts.NewMockInputRunner(ctrl)
ith.Decoder = ts.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *pipeline.PipelinePack, 1)
ith.DecodeChan = make(chan *pipeline.PipelinePack)
ith.MockDecoderSet = ts.NewMockDecoderSet(ctrl)
ith.Msg = getTestMessage()
ith.Pack = pipeline.NewPipelinePack(recycleChan)
c.Specify("can receive a set of metrics", func() {
ith.PackSupply <- ith.Pack
resp := new(http.Response)
resp.Body = &RespCloser{strings.NewReader(awsResponse)}
resp.StatusCode = 200
// Setup the mock response
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Inject(ith.Pack)
serv.EXPECT().Query(gomock.Any(), gomock.Any(), gomock.Any()).Return(resp, nil)
go func() {
input.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
close(input.stopChan)
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "Testing")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "Test")
val, _ := ith.Pack.Message.GetFieldValue("Unit")
c.Expect(val.(string), gs.Equals, "Seconds")
val, _ = ith.Pack.Message.GetFieldValue("SampleCount")
c.Expect(val.(float64), gs.Equals, float64(837721.0))
})
})
c.Specify("A CloudwatchOutput", func() {
mockOutputRunner := ts.NewMockOutputRunner(ctrl)
mockHelper := ts.NewMockPluginHelper(ctrl)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
mockOutputRunner.EXPECT().InChan().Return(inChan)
msg := getTestMessage()
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = msg
pack.Decoded = true
output := new(CloudwatchOutput)
outputConfig := output.ConfigStruct().(*CloudwatchOutputConfig)
outputConfig.Retries = 3
outputConfig.Backlog = 10
outputConfig.Namespace = "Test"
outputConfig.Region = "us-east-1"
err := output.Init(outputConfig)
c.Assume(err, gs.IsNil)
serv := ts.NewMockAWSService(ctrl)
output.cw.Service = serv
c.Specify("can send a batch of metrics", func() {
resp := new(http.Response)
resp.Body = &RespCloser{strings.NewReader(awsSuccessResponse)}
resp.StatusCode = 200
pack.Message.SetPayload(simpleJsonPayload)
serv.EXPECT().Query(gomock.Any(), gomock.Any(), gomock.Any()).Return(resp, nil)
mockOutputRunner.EXPECT().LogMessage(gomock.Any())
finished := make(chan bool)
inChan <- pack
go func() {
output.Run(mockOutputRunner, mockHelper)
finished <- true
}()
<-recycleChan
close(inChan)
<-finished
})
c.Specify("can retry failed operations", func() {
resp := new(http.Response)
resp.Body = &RespCloser{strings.NewReader(awsSuccessResponse)}
resp.StatusCode = 200
err := errors.New("Oops, not working")
pack.Message.SetPayload(simpleJsonPayload)
serv.EXPECT().Query(gomock.Any(), gomock.Any(), gomock.Any()).Times(3).Return(resp, err)
mockOutputRunner.EXPECT().LogMessage(gomock.Any())
mockOutputRunner.EXPECT().LogError(gomock.Any())
finished := make(chan bool)
inChan <- pack
go func() {
output.Run(mockOutputRunner, mockHelper)
finished <- true
}()
<-recycleChan
close(inChan)
<-finished
})
})
}
func FilterSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
fth := NewFilterTestHelper(ctrl)
inChan := make(chan *pipeline.PipelinePack, 1)
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxFilter", func() {
sbFilter := new(SandboxFilter)
sbFilter.SetPipelineConfig(pConfig)
config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
config.MemoryLimit = 64000
config.InstructionLimit = 1000
config.OutputLimit = 1024
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
pack.Message = msg
c.Specify("Uninitialized", func() {
err := sbFilter.ReportMsg(msg)
c.Expect(err, gs.IsNil)
})
c.Specify("Over inject messages from ProcessMessage", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(2)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(2)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/processinject.lua"
config.ModuleDirectory = "../lua/modules"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
var termErr pipeline.TerminatedError
if runtime.GOOS == "windows" {
termErr = pipeline.TerminatedError("process_message() ..\\lua\\testsupport\\processinject.lua:8: inject_payload() exceeded InjectMessage count")
} else {
termErr = pipeline.TerminatedError("process_message() ../lua/testsupport/processinject.lua:8: inject_payload() exceeded InjectMessage count")
}
c.Expect(err.Error(), gs.Equals, termErr.Error())
})
c.Specify("Over inject messages from TimerEvent", func() {
var timer <-chan time.Time
timer = time.Tick(time.Duration(1) * time.Millisecond)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(11)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(11)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/timerinject.lua"
config.ModuleDirectory = "../lua/modules"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
go func() {
time.Sleep(time.Duration(250) * time.Millisecond)
close(inChan)
}()
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
var termErr pipeline.TerminatedError
if runtime.GOOS == "windows" {
termErr = pipeline.TerminatedError("timer_event() ..\\lua\\testsupport\\timerinject.lua:13: inject_payload() exceeded InjectMessage count")
} else {
termErr = pipeline.TerminatedError("timer_event() ../lua/testsupport/timerinject.lua:13: inject_payload() exceeded InjectMessage count")
}
c.Expect(err.Error(), gs.Equals, termErr.Error())
})
c.Specify("Preserves data", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/serialize.lua"
config.ModuleDirectory = "../lua/modules"
config.PreserveData = true
sbFilter.SetName("serialize")
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
c.Specify("process_message error string", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("script provided error message"))
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/process_message_error_string.lua"
config.ModuleDirectory = "../lua/modules"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
})
c.Specify("Over inject messages from the shutdown TimerEvent", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(10)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(10)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(10)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/timerinject.lua"
config.ModuleDirectory = "../lua/modules"
config.TimerEventOnShutdown = true
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
if runtime.GOOS == "windows" {
c.Expect(err.Error(), gs.Equals, "FATAL: timer_event() ..\\lua\\testsupport\\timerinject.lua:13: inject_payload() exceeded InjectMessage count")
} else {
c.Expect(err.Error(), gs.Equals, "FATAL: timer_event() ../lua/testsupport/timerinject.lua:13: inject_payload() exceeded InjectMessage count")
}
})
})
c.Specify("A SandboxManagerFilter", func() {
pConfig.Globals.BaseDir = os.TempDir()
sbxMgrsDir := filepath.Join(pConfig.Globals.BaseDir, "sbxmgrs")
defer func() {
tmpErr := os.RemoveAll(sbxMgrsDir)
c.Expect(tmpErr, gs.IsNil)
}()
sbmFilter := new(SandboxManagerFilter)
sbmFilter.SetPipelineConfig(pConfig)
config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
config.MaxFilters = 1
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
c.Specify("Control message in the past", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UpdateCursor("")
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
pack.BufferedPack = true
pack.DelivErrChan = make(chan error, 1)
inChan <- pack
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
err = <-pack.DelivErrChan
c.Expect(err.Error(), gs.Equals, "Discarded control message: 5 seconds skew")
})
c.Specify("Control message in the future", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UpdateCursor("")
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
pack.BufferedPack = true
pack.DelivErrChan = make(chan error, 1)
inChan <- pack
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
err = <-pack.DelivErrChan
c.Expect(err.Error(), gs.Equals, "Discarded control message: -5 seconds skew")
})
c.Specify("Generates the right default working directory", func() {
sbmFilter.Init(config)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
name := "SandboxManagerFilter"
fth.MockFilterRunner.EXPECT().Name().Return(name)
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
_, err = os.Stat(sbxMgrsDir)
c.Expect(err, gs.IsNil)
})
c.Specify("Sanity check the default sandbox configuration limits", func() {
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
})
c.Specify("Sanity check the user specified sandbox configuration limits", func() {
config.MemoryLimit = 123456
config.InstructionLimit = 4321
config.OutputLimit = 8765
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
})
c.Specify("Creates a SandboxFilter runner", func() {
sbxName := "SandboxFilter"
sbxMgrName := "SandboxManagerFilter"
code := `
function process_message()
inject_payload('{a = "b"}')
return 0
end
`
cfg := `
[%s]
type = "SandboxFilter"
message_matcher = "TRUE"
script_type = "lua"
`
cfg = fmt.Sprintf(cfg, sbxName)
msg.SetPayload(code)
f, err := message.NewField("config", cfg, "toml")
c.Assume(err, gs.IsNil)
msg.AddField(f)
fMatchChan := pConfig.Router().AddFilterMatcher()
errChan := make(chan error)
fth.MockFilterRunner.EXPECT().Name().Return(sbxMgrName)
fullSbxName := fmt.Sprintf("%s-%s", sbxMgrName, sbxName)
fth.MockHelper.EXPECT().Filter(fullSbxName).Return(nil, false)
fth.MockFilterRunner.EXPECT().LogMessage(fmt.Sprintf("Loading: %s", fullSbxName))
sbmFilter.Init(config)
go func() {
err := sbmFilter.loadSandbox(fth.MockFilterRunner, fth.MockHelper, sbxMgrsDir,
msg)
errChan <- err
}()
fMatch := <-fMatchChan
c.Expect(fMatch.MatcherSpecification().String(), gs.Equals, "TRUE")
c.Expect(<-errChan, gs.IsNil)
go func() {
<-pConfig.Router().RemoveFilterMatcher()
}()
ok := pConfig.RemoveFilterRunner(fullSbxName)
c.Expect(ok, gs.IsTrue)
})
})
c.Specify("A Load Average Stats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "loadavg"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/loadavg.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
msg := getTestMessage()
fields := make([]*message.Field, 4)
fields[0], _ = message.NewField("1MinAvg", 0.08, "")
fields[1], _ = message.NewField("5MinAvg", 0.04, "")
fields[2], _ = message.NewField("15MinAvg", 0.02, "")
fields[3], _ = message.NewField("NumProcesses", 5, "")
msg.Fields = fields
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return("loadavg")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with loadavg data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
var i int64
for i = 1; i <= 3; i++ {
// Fill in the data
t := i * 1000000000
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":1,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"1MinAvg","unit":"Count","aggregation":"max"},{"name":"5MinAvg","unit":"Count","aggregation":"max"},{"name":"15MinAvg","unit":"Count","aggregation":"max"},{"name":"NumProcesses","unit":"Count","aggregation":"max"}]}
0.08 0.04 0.02 5
0.08 0.04 0.02 5
0.08 0.04 0.02 5
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("A Memstats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "memstats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/memstats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
msg := getTestMessage()
field_names := []string{"MemFree", "Cached", "Active", "Inactive", "VmallocUsed", "Shmem", "SwapCached", "SwapTotal", "SwapFree"}
fields := make([]*message.Field, len(field_names))
for i, name := range field_names {
fields[i], _ = message.NewField(name, 100, "")
}
msg.Fields = fields
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return("memstats")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with memstats data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
var i int64
for i = 1; i <= 3; i++ {
// Fill in the data
t := i * 1000000000
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":1,"rows":3,"columns":9,"seconds_per_row":1,"column_info":[{"name":"MemFree","unit":"Count","aggregation":"max"},{"name":"Cached","unit":"Count","aggregation":"max"},{"name":"Active","unit":"Count","aggregation":"max"},{"name":"Inactive","unit":"Count","aggregation":"max"},{"name":"VmallocUsed","unit":"Count","aggregation":"max"},{"name":"Shmem","unit":"Count","aggregation":"max"},{"name":"SwapCached","unit":"Count","aggregation":"max"},{"name":"SwapFree","unit":"Count","aggregation":"max"},{"name":"SwapUsed","unit":"Count","aggregation":"max"}]}
100 100 100 100 100 100 100 100 0
100 100 100 100 100 100 100 100 0
100 100 100 100 100 100 100 100 0
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("A diskstats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "diskstats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/diskstats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retMsgChan := make(chan *message.Message, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retMsgChan)
}()
msg := getTestMessage()
field_names := []string{
"WritesCompleted",
"ReadsCompleted",
"SectorsWritten",
"SectorsRead",
"WritesMerged",
"ReadsMerged",
"TimeWriting",
"TimeReading",
"TimeDoingIO",
"WeightedTimeDoingIO",
}
num_fields := len(field_names) + 1
fields := make([]*message.Field, num_fields)
msg.Fields = fields
timeInterval, _ := message.NewField("TickerInterval", 1, "")
fields[num_fields-1] = timeInterval
var fieldVal int64 = 100
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelineConfig().AnyTimes()
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).AnyTimes()
fth.MockFilterRunner.EXPECT().Ticker().Return(timer).AnyTimes()
fth.MockFilterRunner.EXPECT().InChan().Return(inChan).AnyTimes()
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return("diskstats").AnyTimes()
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
msg := pack.Message
pack.Message = new(message.Message)
retMsgChan <- msg
}).Return(true).AnyTimes()
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with diskstats data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
// Iterate 4 times since the first one doesn't actually set the cbuf
// in order to set the delta in the cbuf
var i int64
for i = 1; i <= 4; i++ {
// Fill in the fields
for i, name := range field_names {
fields[i], _ = message.NewField(name, fieldVal, "")
}
// Scale up the value so we can see the delta growing
// by 100 each iteration
fieldVal += i * 100
t := i * 1000000000
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
testExpects := map[string]string{
"Time doing IO": `{"time":2,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"TimeWriting","unit":"ms","aggregation":"max"},{"name":"TimeReading","unit":"ms","aggregation":"max"},{"name":"TimeDoingIO","unit":"ms","aggregation":"max"},{"name":"WeightedTimeDoi","unit":"ms","aggregation":"max"}]}
200 200 200 200
400 400 400 400
700 700 700 700
`,
"Disk Stats": `{"time":2,"rows":3,"columns":6,"seconds_per_row":1,"column_info":[{"name":"WritesCompleted","unit":"per_1_s","aggregation":"none"},{"name":"ReadsCompleted","unit":"per_1_s","aggregation":"none"},{"name":"SectorsWritten","unit":"per_1_s","aggregation":"none"},{"name":"SectorsRead","unit":"per_1_s","aggregation":"none"},{"name":"WritesMerged","unit":"per_1_s","aggregation":"none"},{"name":"ReadsMerged","unit":"per_1_s","aggregation":"none"}]}
100 100 100 100 100 100
200 200 200 200 200 200
300 300 300 300 300 300
`,
}
timer <- time.Now()
for i := 0; i < 2; i++ {
m := <-retMsgChan
name, ok := m.GetFieldValue("payload_name")
c.Assume(ok, gs.IsTrue)
nameVal, ok := name.(string)
c.Assume(ok, gs.IsTrue)
c.Expect(m.GetPayload(), gs.Equals, testExpects[nameVal])
}
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("http_status filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "http_status"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/http_status.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(2)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
field, _ := message.NewField("status", 0, "")
msg := &message.Message{}
msg.SetTimestamp(0)
msg.AddField(field)
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return("http_status")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with http status data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
for i := 0; i <= 6; i++ {
msg.Fields[0].ValueInteger[0] = int64(i * 100) // iterate through the status codes with a bogus status on each end
pack.Message = msg
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":0,"rows":2,"columns":6,"seconds_per_row":1,"column_info":[{"name":"HTTP_100","unit":"count","aggregation":"sum"},{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1 1 1 1 1 2
nan nan nan nan nan nan
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("influx_batch filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "influx_batch"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
errChan := make(chan error, 1)
retMsgChan := make(chan *message.Message, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retMsgChan)
}()
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = &message.Message{}
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321 * 1e9)
pack.Message.SetLogger("Logger")
pack.Message.SetPayload("Payload value lorem ipsum")
pack.Message.SetUuid(uuid.Parse("8e414f01-9d7f-4a48-a5e1-ae92e5954df5"))
f, err := message.NewField("intField", 123, "") // Overwritten in a loop below.
c.Assume(err, gs.IsNil)
err = f.AddValue(456)
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("strField", "0_first", "")
c.Assume(err, gs.IsNil)
err = f.AddValue("0_second")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("strField", "1_first", "")
c.Assume(err, gs.IsNil)
err = f.AddValue("1_second")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("byteField", []byte("first"), "")
c.Assume(err, gs.IsNil)
err = f.AddValue([]byte("second"))
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
fth.MockFilterRunner.EXPECT().Ticker().Return(nil)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return(filter.name)
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retMsgChan <- pack.Message
}).Return(true)
c.Assume(err, gs.IsNil)
conf.ScriptFilename = "../lua/filters/influx_batch.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
c.Specify("encodes a basic message", func() {
conf.Config["flush_count"] = "6"
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
// Having our own reference to the message struct lets us hand it
// back to the pack even after recycling.
msg := pack.Message
for i := 0; i < 6; i++ {
msg.Fields[0].ValueInteger[0] = int64(i * 100)
pack.Message = msg
pack.EncodeMsgBytes()
inChan <- pack
pack = <-recycleChan
}
// timer <- time.Now()
m := <-retMsgChan
// Check the result of the filter's inject
msgStr := `byteField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="first" 54321000
byteField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="second" 54321000
strField_fidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="1_first" 54321000
strField_fidx_1_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="1_second" 54321000
strField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="0_first" 54321000
strField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="0_second" 54321000
intField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value=%d.000000 54321000
intField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value=456.000000 54321000
`
pl := ""
for i := 0; i < 6; i++ {
pl += fmt.Sprintf(msgStr, i*100)
}
c.Expect(m.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
}
func DecoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
// NewPipelineConfig sets up Globals which is needed for the
// pipeline.Prepend*Dir functions to not die during plugin Init().
_ = pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
conf.ScriptType = "lua"
c.Specify("that uses lpeg and inject_message", func() {
dRunner.EXPECT().Name().Return("serialize")
conf.ScriptFilename = "../lua/testsupport/decoder.lua"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("decodes simple messages", func() {
data := "1376389920 debug id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1376389920000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("id")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "2321")
value, ok = pack.Message.GetFieldValue("url")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "example.com")
value, ok = pack.Message.GetFieldValue("item")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "1")
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "1376389920 bogus id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
c.Specify("Preserves data", func() {
conf.ScriptFilename = "../lua/testsupport/serialize.lua"
conf.PreserveData = true
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
decoder.Shutdown()
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("that only uses write_message", func() {
conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
dRunner.EXPECT().Name().Return("write_message")
err := decoder.Init(conf)
decoder.SetDecoderRunner(dRunner)
c.Assume(err, gs.IsNil)
c.Specify("adds a string field to the message", func() {
data := "string field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(string), gs.Equals, "foo")
})
c.Specify("adds a numeric field to the message", func() {
data := "num field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(float64), gs.Equals, float64(1))
})
c.Specify("adds a boolean field to the message", func() {
data := "bool field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(bool), gs.Equals, true)
})
c.Specify("sets type and payload", func() {
data := "set type and payload"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
})
c.Specify("sets field value with representation", func() {
data := "set field value with representation"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("rep")
c.Expect(len(fields), gs.Equals, 1)
field := fields[0]
values := field.GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "foo")
c.Expect(field.GetRepresentation(), gs.Equals, "representation")
})
c.Specify("sets multiple field string values", func() {
data := "set multiple field string values"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("multi")
c.Expect(len(fields), gs.Equals, 2)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "first")
values = fields[1].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "second")
})
c.Specify("sets field string array value", func() {
data := "set field string array value"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("array")
c.Expect(len(fields), gs.Equals, 1)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 2)
c.Expect(values[0], gs.Equals, "first")
c.Expect(values[1], gs.Equals, "second")
})
})
})
c.Specify("A Multipack SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
conf.ScriptType = "lua"
supply := make(chan *pipeline.PipelinePack, 3)
pack := pipeline.NewPipelinePack(supply)
pack.Message = getTestMessage()
pack1 := pipeline.NewPipelinePack(supply)
pack2 := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
c.Specify("decodes into multiple packs", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
gomock.InOrder(
dRunner.EXPECT().NewPack().Return(pack1),
dRunner.EXPECT().NewPack().Return(pack2),
)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 3)
c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")
for i := 0; i < 1; i++ {
c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))
}
decoder.Shutdown()
})
})
c.Specify("Nginx access log decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.ScriptType = "lua"
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("rsyslog decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.ScriptType = "lua"
conf.Config = make(map[string]interface{})
conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
conf.Config["tz"] = "America/Los_Angeles"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost kernel: imklog 5.8.6, log source = /proc/kmsg started.\n"
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392065938000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
c.Expect(pack.Message.GetPayload(), gs.Equals, "imklog 5.8.6, log source = /proc/kmsg started.")
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("syslogtag")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "kernel:")
value, ok = pack.Message.GetFieldValue("syslogfacility")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(3))
value, ok = pack.Message.GetFieldValue("timegenerated")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(1392065939000000000))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
}
func DashboardOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
dashboardOutput := new(DashboardOutput)
dashboardOutput.pConfig = pConfig
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)
errChan := make(chan error, 1)
startOutput := func() {
go func() {
errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
}()
}
if runtime.GOOS != "windows" {
c.Specify("A DashboardOutput", func() {
tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
c.Assume(err, gs.IsNil)
config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
config.WorkingDirectory = tmpdir
c.Specify("Init halts if basedirectory is not writable", func() {
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
err = dashboardOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("that is running", func() {
startedChan := make(chan bool, 1)
defer close(startedChan)
ts := httptest.NewUnstartedServer(nil)
dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
ts.Start()
startedChan <- true
return nil
}
ticker := make(chan time.Time)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)
err := os.MkdirAll(tmpdir, 0700)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Noop
})
c.Specify("sets custom http headers", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err = dashboardOutput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = dashboardOutput.server
startOutput()
inChan <- pack
<-startedChan
resp, err := http.Get(ts.URL)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
// Verify headers are there
eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
c.Expect(eq, gs.IsTrue)
eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
c.Expect(eq, gs.IsTrue)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
ts.Close()
})
})
}
}
func NsqOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
var wg sync.WaitGroup
errChan := make(chan error, 1)
defer close(errChan)
c.Specify("A nsq output", func() {
output := new(NsqOutput)
oth := plugins_ts.NewOutputTestHelper(ctrl)
config := output.ConfigStruct().(*NsqOutputConfig)
config.Topic = "test"
startOutput := func() {
wg.Add(1)
go func() {
errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
}
var mockProducer *MockProducer
output.newProducer = func(addr string, config *nsq.Config) (Producer, error) {
mockProducer, _ = NewMockProducer(addr, config)
return mockProducer, nil
}
msg := pipeline_ts.GetTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
c.Specify("requires at least one address", func() {
err := output.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
config.Addresses = []string{"test.com:4150"}
c.Specify("requires an encoder", func() {
err := output.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(nil)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
encoder := new(plugins.PayloadEncoder)
encoder.Init(&plugins.PayloadEncoderConfig{PrefixTs: false})
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
inChan := make(chan *pipeline.PipelinePack, 1)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
c.Specify("publishes a message with the configured topic", func() {
err := output.Init(config)
c.Assume(err, gs.IsNil)
startOutput()
inChan <- pack
transaction := <-mockProducer.respChan
c.Expect(transaction.Error, gs.IsNil)
close(inChan)
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
})
})
})
}
func UnixOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
c.Specify("A UnixOutput", func() {
unixOutput := new(UdpOutput)
config := unixOutput.ConfigStruct().(*UdpOutputConfig)
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(new(plugins.PayloadEncoderConfig))
inChan := make(chan *pipeline.PipelinePack, 1)
rChan := make(chan *pipeline.PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
payload := "Write me out to the network."
msg.SetPayload(payload)
pack := pipeline.NewPipelinePack(rChan)
pack.Message = msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
c.Specify("using UDP", func() {
addr := "@/heka/unix-out"
config.Address = addr
ch := make(chan string, 1)
var wg sync.WaitGroup
var rAddr net.Addr
collectData := func() {
conn, err := net.ListenPacket("unix", addr)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := unixOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = "@/heka/unix-out"
err := unixOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "unix")
c.Expect(rAddr.String(), gs.Equals, "@/heka/unix-out")
close(inChan)
wg.Wait()
})
})
})
}
func OutputSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := pipeline.NewPipelineConfig(nil)
inChan := make(chan *pipeline.PipelinePack, 1)
timer := make(chan time.Time, 1)
c.Specify("A SandboxOutput", func() {
output := new(SandboxOutput)
output.SetPipelineConfig(pConfig)
conf := output.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/output.lua"
conf.ModuleDirectory = "../lua/modules;../lua/testsupport/modules"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
data := "1376389920 debug id=2321 url=example.com item=1"
pack.Message.SetPayload(data)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Ticker().Return(timer)
c.Specify("writes a payload to file", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
inChan <- pack
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Assume(err, gs.IsNil)
tmpFile, err := os.Open("output.lua.txt")
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, data)
c.Expect(output.processMessageCount, gs.Equals, int64(1))
c.Expect(output.processMessageSamples, gs.Equals, int64(1))
c.Expect(output.processMessageFailures, gs.Equals, int64(0))
})
c.Specify("failure processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
pack.Message.SetPayload("FAILURE")
pack.BufferedPack = true
pack.DelivErrChan = make(chan error, 1)
inChan <- pack
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Assume(err, gs.IsNil)
c.Expect(output.processMessageFailures, gs.Equals, int64(1))
err = <-pack.DelivErrChan
c.Expect(err.Error(), gs.Equals, "failure message")
})
c.Specify("user abort processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
e := errors.New("FATAL: user abort")
pack.Message.SetPayload("USERABORT")
inChan <- pack
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err.Error(), gs.Equals, e.Error())
})
c.Specify("fatal error processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
pack.Message.SetPayload("FATAL")
inChan <- pack
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("fatal error in timer_event", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
timer <- time.Now()
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("fatal error in shutdown timer_event", func() {
conf.TimerEventOnShutdown = true
err := output.Init(conf)
c.Expect(err, gs.IsNil)
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func getTestPipelinePack() *pipeline.PipelinePack {
return pipeline.NewPipelinePack(&config)
}
func NagiosOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
c.Specify("A NagiosOutput", func() {
output := new(NagiosOutput)
config := output.ConfigStruct().(*NagiosOutputConfig)
config.Url = "http://localhost:55580/foo/bar"
mockOutputRunner := pipelinemock.NewMockOutputRunner(ctrl)
mockHelper := pipelinemock.NewMockPluginHelper(ctrl)
inChan := make(chan *pipeline.PipelinePack)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
msg := pipeline_ts.GetTestMessage()
pack.Message = msg
var req *http.Request
var outputWg, reqWg sync.WaitGroup
run := func() {
mockOutputRunner.EXPECT().InChan().Return(inChan)
mockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
output.Run(mockOutputRunner, mockHelper)
outputWg.Done()
}
const payload = "something"
c.Specify("using HTTP transport", func() {
// Spin up an HTTP listener.
listener, err := net.Listen("tcp", "127.0.0.1:55580")
c.Assume(err, gs.IsNil)
mux := http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Connection", "close")
r.ParseForm()
req = r
listener.Close()
reqWg.Done()
})
go http.Serve(listener, mux)
c.Specify("sends a valid HTTP POST", func() {
err = output.Init(config)
c.Assume(err, gs.IsNil)
outputWg.Add(1)
go run()
msg.SetPayload("OK:" + payload)
reqWg.Add(1)
inChan <- pack
close(inChan)
outputWg.Wait()
reqWg.Wait()
c.Expect(req.FormValue("plugin_output"), gs.Equals, payload)
c.Expect(req.FormValue("plugin_state"), gs.Equals, "0")
})
c.Specify("correctly maps alternate state", func() {
err = output.Init(config)
c.Assume(err, gs.IsNil)
outputWg.Add(1)
go run()
msg.SetPayload("CRITICAL:" + payload)
reqWg.Add(1)
inChan <- pack
close(inChan)
outputWg.Wait()
reqWg.Wait()
c.Expect(req.FormValue("plugin_output"), gs.Equals, payload)
c.Expect(req.FormValue("plugin_state"), gs.Equals, "2")
})
})
if runtime.GOOS != "windows" {
outPath := filepath.Join(os.TempDir(), "heka-nagios-test-output.txt")
echoFile := fmt.Sprintf(echoFileTmpl, outPath)
c.Specify("using a send_ncsa binary", func() {
binPath := pipeline_ts.WriteStringToTmpFile(echoFile)
os.Chmod(binPath, 0700)
defer func() {
os.Remove(binPath)
os.Remove(outPath)
}()
config.SendNscaBin = binPath
config.SendNscaArgs = []string{"arg1", "arg2"}
err := output.Init(config)
c.Assume(err, gs.IsNil)
outputWg.Add(1)
go run()
c.Specify("sends args and the right data through", func() {
msg.SetPayload("OK:" + payload)
inChan <- pack
close(inChan)
outputWg.Wait()
outFile, err := os.Open(outPath)
c.Expect(err, gs.IsNil)
reader := bufio.NewReader(outFile)
line, _, err := reader.ReadLine()
c.Expect(err, gs.IsNil)
c.Expect(string(line), gs.Equals, strings.Join(config.SendNscaArgs, " "))
line, _, err = reader.ReadLine()
c.Expect(err, gs.IsNil)
c.Expect(string(line), gs.Equals, "my.host.name GoSpec 0 "+payload)
})
c.Specify("correctly maps alternate state", func() {
msg.SetPayload("WARNING:" + payload)
inChan <- pack
close(inChan)
outputWg.Wait()
outFile, err := os.Open(outPath)
c.Expect(err, gs.IsNil)
reader := bufio.NewReader(outFile)
line, _, err := reader.ReadLine()
c.Expect(err, gs.IsNil)
c.Expect(string(line), gs.Equals, strings.Join(config.SendNscaArgs, " "))
line, _, err = reader.ReadLine()
c.Expect(err, gs.IsNil)
c.Expect(string(line), gs.Equals, "my.host.name GoSpec 1 "+payload)
})
})
}
})
}
func DecoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
// NewPipelineConfig sets up Globals which is needed for the
// pipeline.Prepend*Dir functions to not die during plugin Init().
_ = pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
c.Specify("that uses lpeg and inject_message", func() {
dRunner.EXPECT().Name().Return("serialize")
conf.ScriptFilename = "../lua/testsupport/decoder.lua"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("decodes simple messages", func() {
data := "1376389920 debug id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1376389920000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("id")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "2321")
value, ok = pack.Message.GetFieldValue("url")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "example.com")
value, ok = pack.Message.GetFieldValue("item")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "1")
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "1376389920 bogus id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
c.Specify("Preserves data", func() {
conf.ScriptFilename = "../lua/testsupport/serialize.lua"
conf.PreserveData = true
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
decoder.Shutdown()
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("that only uses write_message", func() {
conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
dRunner.EXPECT().Name().Return("write_message")
err := decoder.Init(conf)
decoder.SetDecoderRunner(dRunner)
c.Assume(err, gs.IsNil)
c.Specify("adds a string field to the message", func() {
data := "string field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(string), gs.Equals, "foo")
})
c.Specify("adds a numeric field to the message", func() {
data := "num field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(float64), gs.Equals, float64(1))
})
c.Specify("adds a boolean field to the message", func() {
data := "bool field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(bool), gs.Equals, true)
})
c.Specify("sets type and payload", func() {
data := "set type and payload"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
})
c.Specify("sets field value with representation", func() {
data := "set field value with representation"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("rep")
c.Expect(len(fields), gs.Equals, 1)
field := fields[0]
values := field.GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "foo")
c.Expect(field.GetRepresentation(), gs.Equals, "representation")
})
c.Specify("sets multiple field string values", func() {
data := "set multiple field string values"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("multi")
c.Expect(len(fields), gs.Equals, 2)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "first")
values = fields[1].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "second")
})
c.Specify("sets field string array value", func() {
data := "set field string array value"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("array")
c.Expect(len(fields), gs.Equals, 1)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 2)
c.Expect(values[0], gs.Equals, "first")
c.Expect(values[1], gs.Equals, "second")
})
})
})
c.Specify("A Multipack SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
supply := make(chan *pipeline.PipelinePack, 3)
pack := pipeline.NewPipelinePack(supply)
pack.Message = getTestMessage()
pack1 := pipeline.NewPipelinePack(supply)
pack2 := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
c.Specify("decodes into multiple packs", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
gomock.InOrder(
dRunner.EXPECT().NewPack().Return(pack1),
dRunner.EXPECT().NewPack().Return(pack2),
)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 3)
c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")
for i := 0; i < 1; i++ {
c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))
}
decoder.Shutdown()
})
})
c.Specify("Nginx access log decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Apache access log decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/apache_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("rsyslog decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["type"] = "MyTestFormat"
conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
conf.Config["tz"] = "America/Los_Angeles"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n"
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392065938000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322))
c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.")
c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"])
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("programname")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "widget")
value, ok = pack.Message.GetFieldValue("syslogfacility")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(3))
value, ok = pack.Message.GetFieldValue("timegenerated")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(1392065939000000000))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("mysql decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["truncate_sql"] = int64(5)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decode standard slow query log", func() {
data := `# User@Host: syncrw[syncrw] @ [127.0.0.1]
# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773
use widget;
SET last_insert_id=999,insert_id=1000,timestamp=1399500744;
# administrator command: do something
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;
`
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1399500744000000000))
c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query")
decoder.Shutdown()
})
})
c.Specify("mariadb decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["truncate_sql"] = int64(5)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decode standard slow query log", func() {
data := `# User@Host: syncrw[syncrw] @ [127.0.0.1]
# Thread_id: 110804 Schema: weave0 QC_hit: No
# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;
`
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1399500744000000000))
c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query")
decoder.Shutdown()
})
})
}
func HttpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
var (
reqMethod string
reqBody string
reqHeader http.Header
handleWg sync.WaitGroup
runWg sync.WaitGroup
delay bool
)
handler := new(_test_handler)
handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) {
defer handleWg.Done()
if delay {
time.Sleep(time.Duration(2) * time.Millisecond)
}
// Capture request body for test assertions.
p := make([]byte, req.ContentLength)
_, err := req.Body.Read(p)
c.Expect(err.Error(), gs.Equals, "EOF")
reqBody = string(p)
// Capture request method and headers for test assertions.
reqMethod = req.Method
reqHeader = req.Header
// Respond with either a response body or a response code.
if len(handler.respBody) > 0 {
rw.Write([]byte(handler.respBody))
} else {
rw.WriteHeader(handler.respCode)
}
flusher := rw.(http.Flusher)
flusher.Flush()
}
oth := ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encConfig := new(plugins.PayloadEncoderConfig)
err := encoder.Init(encConfig)
c.Expect(err, gs.IsNil)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
c.Specify("An HttpOutput", func() {
httpOutput := new(HttpOutput)
config := httpOutput.ConfigStruct().(*HttpOutputConfig)
c.Specify("barfs on bogus URLs", func() {
config.Address = "one-two-three-four"
err := httpOutput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
server := httptest.NewServer(handler)
defer server.Close()
runOutput := func() {
httpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
runWg.Done()
}
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
payload := "this is the payload"
pack.Message.SetPayload(payload)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(
[]byte(payload), nil)
config.Address = server.URL
handler.respBody = "Response Body"
c.Specify("makes http POST requests by default", func() {
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(reqMethod, gs.Equals, "POST")
})
c.Specify("makes http PUT requests", func() {
config.Method = "put"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(reqMethod, gs.Equals, "PUT")
})
c.Specify("makes http GET requests", func() {
config.Method = "get"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, "")
c.Expect(reqMethod, gs.Equals, "GET")
})
c.Specify("correctly passes headers along", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"]))
c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"]))
})
c.Specify("uses http auth when specified", func() {
config.Username = "******"
config.Password = "******"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
auth := reqHeader.Get("Authorization")
c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue)
decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:])
c.Expect(err, gs.IsNil)
c.Expect(string(decodedAuth), gs.Equals, "user:pass")
})
c.Specify("logs error responses", func() {
handler.respBody = ""
handler.respCode = 500
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
var errMsg string
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
func(err error) {
errMsg = err.Error()
})
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(strings.HasPrefix(errMsg,
"HTTP Error code returned: 500"), gs.IsTrue)
})
c.Specify("honors http timeout interval", func() {
config.HttpTimeout = 1 // 1 millisecond
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
var errMsg string
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
func(err error) {
errMsg = err.Error()
})
delay = true
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(strings.Contains(errMsg, "use of closed network connection"),
gs.IsTrue)
})
})
})
}
func EncoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
// NewPipelineConfig sets up Globals which is needed for the
// pipeline.Prepend*Dir functions to not die during plugin Init().
_ = pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxEncoder", func() {
encoder := new(SandboxEncoder)
conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
pack.Message.SetPayload("original")
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321)
pack.Message.SetUuid(uuid.NewRandom())
var (
result []byte
err error
)
c.Specify("emits JSON correctly", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_json.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = json.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
c.Expect(msg.GetPid(), gs.Equals, int32(12345))
c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
c.Expect(msg.GetHostname(), gs.Equals, "hostname")
c.Expect(msg.GetPayload(), gs.Equals, "original")
c.Expect(msg.GetType(), gs.Equals, "my_type")
})
c.Specify("emits text correctly", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_text.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
c.Expect(string(result), gs.Equals, "Prefixed original")
})
c.Specify("emits protobuf correctly", func() {
c.Specify("when inject_message is used", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_protobuf.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = proto.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
c.Expect(msg.GetPid(), gs.Equals, int32(12345))
c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
c.Expect(msg.GetHostname(), gs.Equals, "hostname")
c.Expect(msg.GetPayload(), gs.Equals, "mutated")
c.Expect(msg.GetType(), gs.Equals, "after")
})
c.Specify("when `write_message` is used", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_writemessage.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = proto.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetPayload(), gs.Equals, "mutated payload")
c.Expect(pack.Message.GetPayload(), gs.Equals, "original")
})
})
})
}
func getTestPack() *pipeline.PipelinePack {
pack := pipeline.NewPipelinePack(nil)
pack.Message = getTestMessage()
return pack
}
func FilterSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
fth := NewFilterTestHelper(ctrl)
inChan := make(chan *pipeline.PipelinePack, 1)
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxFilter", func() {
sbFilter := new(SandboxFilter)
sbFilter.SetPipelineConfig(pConfig)
config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
config.MemoryLimit = 32000
config.InstructionLimit = 1000
config.OutputLimit = 1024
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("Uninitialized", func() {
err := sbFilter.ReportMsg(msg)
c.Expect(err, gs.IsNil)
})
c.Specify("Over inject messages from ProcessMessage", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(2)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(2)
config.ScriptFilename = "../lua/testsupport/processinject.lua"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
termErr := pipeline.TerminatedError("exceeded InjectMessage count")
c.Expect(err.Error(), gs.Equals, termErr.Error())
})
c.Specify("Over inject messages from TimerEvent", func() {
var timer <-chan time.Time
timer = time.Tick(time.Duration(1) * time.Millisecond)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(11)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(11)
config.ScriptFilename = "../lua/testsupport/timerinject.lua"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
go func() {
time.Sleep(time.Duration(250) * time.Millisecond)
close(inChan)
}()
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
termErr := pipeline.TerminatedError("exceeded InjectMessage count")
c.Expect(err.Error(), gs.Equals, termErr.Error())
})
c.Specify("Preserves data", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
config.ScriptFilename = "../lua/testsupport/serialize.lua"
config.PreserveData = true
sbFilter.SetName("serialize")
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("A SandboxManagerFilter", func() {
pConfig.Globals.BaseDir = os.TempDir()
sbxMgrsDir := filepath.Join(pConfig.Globals.BaseDir, "sbxmgrs")
defer func() {
tmpErr := os.RemoveAll(sbxMgrsDir)
c.Expect(tmpErr, gs.IsNil)
}()
sbmFilter := new(SandboxManagerFilter)
sbmFilter.SetPipelineConfig(pConfig)
config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
config.MaxFilters = 1
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("Control message in the past", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: 5 seconds skew"))
inChan <- pack
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
})
c.Specify("Control message in the future", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: -5 seconds skew"))
inChan <- pack
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
})
c.Specify("Generates the right default working directory", func() {
sbmFilter.Init(config)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
name := "SandboxManagerFilter"
fth.MockFilterRunner.EXPECT().Name().Return(name)
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
_, err = os.Stat(sbxMgrsDir)
c.Expect(err, gs.IsNil)
})
c.Specify("Sanity check the default sandbox configuration limits", func() {
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
})
c.Specify("Sanity check the user specified sandbox configuration limits", func() {
config.MemoryLimit = 123456
config.InstructionLimit = 4321
config.OutputLimit = 8765
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
})
c.Specify("Creates a SandboxFilter runner", func() {
sbxName := "SandboxFilter"
sbxMgrName := "SandboxManagerFilter"
code := `
require("cjson")
function process_message()
inject_payload(cjson.encode({a = "b"}))
return 0
end
`
cfg := `
[%s]
type = "SandboxFilter"
message_matcher = "TRUE"
script_type = "lua"
`
cfg = fmt.Sprintf(cfg, sbxName)
msg.SetPayload(code)
f, err := message.NewField("config", cfg, "toml")
c.Assume(err, gs.IsNil)
msg.AddField(f)
fMatchChan := pConfig.Router().AddFilterMatcher()
errChan := make(chan error)
fth.MockFilterRunner.EXPECT().Name().Return(sbxMgrName)
fullSbxName := fmt.Sprintf("%s-%s", sbxMgrName, sbxName)
fth.MockHelper.EXPECT().Filter(fullSbxName).Return(nil, false)
fth.MockFilterRunner.EXPECT().LogMessage(fmt.Sprintf("Loading: %s", fullSbxName))
sbmFilter.Init(config)
go func() {
err := sbmFilter.loadSandbox(fth.MockFilterRunner, fth.MockHelper, sbxMgrsDir,
msg)
errChan <- err
}()
fMatch := <-fMatchChan
c.Expect(fMatch.MatcherSpecification().String(), gs.Equals, "TRUE")
c.Expect(<-errChan, gs.IsNil)
go func() {
<-pConfig.Router().RemoveFilterMatcher()
}()
ok := pConfig.RemoveFilterRunner(fullSbxName)
c.Expect(ok, gs.IsTrue)
})
})
c.Specify("A Cpu Stats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "cpustats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/cpustats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
msg := getTestMessage()
fields := make([]*message.Field, 4)
fields[0], _ = message.NewField("1MinAvg", 0.08, "")
fields[1], _ = message.NewField("5MinAvg", 0.04, "")
fields[2], _ = message.NewField("15MinAvg", 0.02, "")
fields[3], _ = message.NewField("NumProcesses", 5, "")
msg.Fields = fields
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("cpustats")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with cpuload data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
for i := 1; i <= 3; i++ {
// Fill in the data
t := int64(i * 1000000000)
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":1,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"1MinAvg","unit":"Count","aggregation":"max"},{"name":"5MinAvg","unit":"Count","aggregation":"max"},{"name":"15MinAvg","unit":"Count","aggregation":"max"},{"name":"NumProcesses","unit":"Count","aggregation":"max"}]}
0.08 0.04 0.02 5
0.08 0.04 0.02 5
0.08 0.04 0.02 5
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("A Memstats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "memstats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/memstats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
msg := getTestMessage()
field_names := []string{"MemFree", "Cached", "Active", "Inactive", "VmallocUsed", "Shmem", "SwapCached", "SwapTotal", "SwapFree"}
fields := make([]*message.Field, len(field_names))
for i, name := range field_names {
fields[i], _ = message.NewField(name, 100, "")
}
msg.Fields = fields
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("memstats")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with memstats data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
for i := 1; i <= 3; i++ {
// Fill in the data
t := int64(i * 1000000000)
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":1,"rows":3,"columns":9,"seconds_per_row":1,"column_info":[{"name":"MemFree","unit":"Count","aggregation":"max"},{"name":"Cached","unit":"Count","aggregation":"max"},{"name":"Active","unit":"Count","aggregation":"max"},{"name":"Inactive","unit":"Count","aggregation":"max"},{"name":"VmallocUsed","unit":"Count","aggregation":"max"},{"name":"Shmem","unit":"Count","aggregation":"max"},{"name":"SwapCached","unit":"Count","aggregation":"max"},{"name":"SwapFree","unit":"Count","aggregation":"max"},{"name":"SwapUsed","unit":"Count","aggregation":"max"}]}
100 100 100 100 100 100 100 100 0
100 100 100 100 100 100 100 100 0
100 100 100 100 100 100 100 100 0
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("A diskstats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "diskstats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/diskstats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retMsgChan := make(chan *message.Message, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retMsgChan)
}()
msg := getTestMessage()
field_names := []string{
"WritesCompleted",
"ReadsCompleted",
"SectorsWritten",
"SectorsRead",
"WritesMerged",
"ReadsMerged",
"TimeWriting",
"TimeReading",
"TimeDoingIO",
"WeightedTimeDoingIO",
}
num_fields := len(field_names) + 1
fields := make([]*message.Field, num_fields)
msg.Fields = fields
timeInterval, _ := message.NewField("TickerInterval", 1, "")
fields[num_fields-1] = timeInterval
fieldVal := 100
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelineConfig().AnyTimes()
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).AnyTimes()
fth.MockFilterRunner.EXPECT().Ticker().Return(timer).AnyTimes()
fth.MockFilterRunner.EXPECT().InChan().Return(inChan).AnyTimes()
fth.MockFilterRunner.EXPECT().Name().Return("diskstats").AnyTimes()
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
msg := pack.Message
pack.Message = new(message.Message)
retMsgChan <- msg
}).Return(true).AnyTimes()
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with diskstats data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
// Iterate 4 times since the first one doesn't actually set the cbuf
// in order to set the delta in the cbuf
for i := 1; i <= 4; i++ {
// Fill in the fields
for i, name := range field_names {
fields[i], _ = message.NewField(name, fieldVal, "")
}
// Scale up the value so we can see the delta growing
// by 100 each iteration
fieldVal += i * 100
t := int64(i * 1000000000)
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
testExpects := map[string]string{
"Time doing IO": `{"time":2,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"TimeWriting","unit":"ms","aggregation":"max"},{"name":"TimeReading","unit":"ms","aggregation":"max"},{"name":"TimeDoingIO","unit":"ms","aggregation":"max"},{"name":"WeightedTimeDoi","unit":"ms","aggregation":"max"}]}
200 200 200 200
400 400 400 400
700 700 700 700
`,
"Disk Stats": `{"time":2,"rows":3,"columns":6,"seconds_per_row":1,"column_info":[{"name":"WritesCompleted","unit":"per_1_s","aggregation":"none"},{"name":"ReadsCompleted","unit":"per_1_s","aggregation":"none"},{"name":"SectorsWritten","unit":"per_1_s","aggregation":"none"},{"name":"SectorsRead","unit":"per_1_s","aggregation":"none"},{"name":"WritesMerged","unit":"per_1_s","aggregation":"none"},{"name":"ReadsMerged","unit":"per_1_s","aggregation":"none"}]}
100 100 100 100 100 100
200 200 200 200 200 200
300 300 300 300 300 300
`,
}
timer <- time.Now()
for i := 0; i < 2; i++ {
m := <-retMsgChan
name, ok := m.GetFieldValue("payload_name")
c.Assume(ok, gs.IsTrue)
nameVal, ok := name.(string)
c.Assume(ok, gs.IsTrue)
c.Expect(m.GetPayload(), gs.Equals, testExpects[nameVal])
}
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("http_status filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "http_status"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/http_status.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(2)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
field, _ := message.NewField("status", 0, "")
msg := &message.Message{}
msg.SetTimestamp(0)
msg.AddField(field)
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("http_status")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with http status data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
for i := 0; i <= 6; i++ {
msg.Fields[0].ValueInteger[0] = int64(i * 100) // iterate through the status codes with a bogus status on each end
pack.Message = msg
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":0,"rows":2,"columns":6,"seconds_per_row":1,"column_info":[{"name":"HTTP_100","unit":"count","aggregation":"sum"},{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1 1 1 1 1 2
nan nan nan nan nan nan
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
}
func EncoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
// NewPipelineConfig sets up Globals which is needed for the
// pipeline.Prepend*Dir functions to not die during plugin Init().
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxEncoder", func() {
encoder := new(SandboxEncoder)
encoder.SetPipelineConfig(pConfig)
conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
pack.Message.SetPayload("original")
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321)
pack.Message.SetUuid(uuid.NewRandom())
var (
result []byte
err error
)
c.Specify("emits JSON correctly", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_json.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = json.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
c.Expect(msg.GetPid(), gs.Equals, int32(12345))
c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
c.Expect(msg.GetHostname(), gs.Equals, "hostname")
c.Expect(msg.GetPayload(), gs.Equals, "original")
c.Expect(msg.GetType(), gs.Equals, "my_type")
})
c.Specify("emits text correctly", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_text.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
c.Expect(string(result), gs.Equals, "Prefixed original")
})
c.Specify("emits protobuf correctly", func() {
c.Specify("when inject_message is used", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_protobuf.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = proto.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
c.Expect(msg.GetPid(), gs.Equals, int32(12345))
c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
c.Expect(msg.GetHostname(), gs.Equals, "hostname")
c.Expect(msg.GetPayload(), gs.Equals, "mutated")
c.Expect(msg.GetType(), gs.Equals, "after")
})
c.Specify("when `write_message` is used", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_writemessage.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = proto.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetPayload(), gs.Equals, "mutated payload")
c.Expect(pack.Message.GetPayload(), gs.Equals, "original")
})
})
})
c.Specify("cbuf librato encoder", func() {
encoder := new(SandboxEncoder)
encoder.SetPipelineConfig(pConfig)
conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321)
pack.Message.SetUuid(uuid.NewRandom())
var (
result []byte
err error
)
conf.ScriptFilename = "../lua/encoders/cbuf_librato.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.Config = make(map[string]interface{})
err = encoder.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("encodes cbuf data", func() {
payload := `{"time":1410823460,"rows":5,"columns":5,"seconds_per_row":5,"column_info":[{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1 2 3 4 5
6 7 8 9 10
11 12 13 14 15
16 17 18 19 20
21 22 23 24 25
`
pack.Message.SetPayload(payload)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `{"gauges":[{"value":1,"measure_time":1410823460,"name":"HTTP_200","source":"hostname"},{"value":2,"measure_time":1410823460,"name":"HTTP_300","source":"hostname"},{"value":3,"measure_time":1410823460,"name":"HTTP_400","source":"hostname"},{"value":4,"measure_time":1410823460,"name":"HTTP_500","source":"hostname"},{"value":5,"measure_time":1410823460,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":6,"measure_time":1410823465,"name":"HTTP_200","source":"hostname"},{"value":7,"measure_time":1410823465,"name":"HTTP_300","source":"hostname"},{"value":8,"measure_time":1410823465,"name":"HTTP_400","source":"hostname"},{"value":9,"measure_time":1410823465,"name":"HTTP_500","source":"hostname"},{"value":10,"measure_time":1410823465,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":11,"measure_time":1410823470,"name":"HTTP_200","source":"hostname"},{"value":12,"measure_time":1410823470,"name":"HTTP_300","source":"hostname"},{"value":13,"measure_time":1410823470,"name":"HTTP_400","source":"hostname"},{"value":14,"measure_time":1410823470,"name":"HTTP_500","source":"hostname"},{"value":15,"measure_time":1410823470,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":16,"measure_time":1410823475,"name":"HTTP_200","source":"hostname"},{"value":17,"measure_time":1410823475,"name":"HTTP_300","source":"hostname"},{"value":18,"measure_time":1410823475,"name":"HTTP_400","source":"hostname"},{"value":19,"measure_time":1410823475,"name":"HTTP_500","source":"hostname"},{"value":20,"measure_time":1410823475,"name":"HTTP_UNKNOWN","source":"hostname"}]}`
c.Expect(string(result), gs.Equals, expected)
c.Specify("and correctly advances", func() {
payload := `{"time":1410823475,"rows":5,"columns":5,"seconds_per_row":5,"column_info":[{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
16 17 18 19 20
21 22 23 24 25
1 2 3 4 5
6 nan 8 nan 10
5 4 3 2 1
`
pack.Message.SetPayload(payload)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `{"gauges":[{"value":21,"measure_time":1410823480,"name":"HTTP_200","source":"hostname"},{"value":22,"measure_time":1410823480,"name":"HTTP_300","source":"hostname"},{"value":23,"measure_time":1410823480,"name":"HTTP_400","source":"hostname"},{"value":24,"measure_time":1410823480,"name":"HTTP_500","source":"hostname"},{"value":25,"measure_time":1410823480,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":1,"measure_time":1410823485,"name":"HTTP_200","source":"hostname"},{"value":2,"measure_time":1410823485,"name":"HTTP_300","source":"hostname"},{"value":3,"measure_time":1410823485,"name":"HTTP_400","source":"hostname"},{"value":4,"measure_time":1410823485,"name":"HTTP_500","source":"hostname"},{"value":5,"measure_time":1410823485,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":6,"measure_time":1410823490,"name":"HTTP_200","source":"hostname"},{"value":8,"measure_time":1410823490,"name":"HTTP_400","source":"hostname"},{"value":10,"measure_time":1410823490,"name":"HTTP_UNKNOWN","source":"hostname"}]}`
c.Expect(string(result), gs.Equals, expected)
})
})
})
c.Specify("schema influx encoder", func() {
encoder := new(SandboxEncoder)
encoder.SetPipelineConfig(pConfig)
conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321 * 1e9)
pack.Message.SetLogger("Logger")
pack.Message.SetPayload("Payload value lorem ipsum")
f, err := message.NewField("intField", 123, "")
c.Assume(err, gs.IsNil)
err = f.AddValue(456)
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("strField", "0_first", "")
c.Assume(err, gs.IsNil)
err = f.AddValue("0_second")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("strField", "1_first", "")
c.Assume(err, gs.IsNil)
err = f.AddValue("1_second")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("byteField", []byte("first"), "")
c.Assume(err, gs.IsNil)
err = f.AddValue([]byte("second"))
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
conf.ScriptFilename = "../lua/encoders/schema_influx.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.Config = make(map[string]interface{})
c.Specify("encodes a basic message", func() {
err = encoder.Init(conf)
c.Assume(err, gs.IsNil)
result, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `[{"points":[[54321000,"my_type","Payload value lorem ipsum","hostname",12345,"Logger",4,"",[123,456],["0_first","0_second"],["1_first","1_second"]]],"name":"series","columns":["time","Type","Payload","Hostname","Pid","Logger","Severity","EnvVersion","intField","strField","strField2"]}]`
c.Expect(string(result), gs.Equals, expected)
})
c.Specify("interpolates series name correctly", func() {
conf.Config["series"] = "series.%{Pid}.%{Type}.%{strField}.%{intField}"
err = encoder.Init(conf)
c.Assume(err, gs.IsNil)
result, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `[{"points":[[54321000,"my_type","Payload value lorem ipsum","hostname",12345,"Logger",4,"",[123,456],["0_first","0_second"],["1_first","1_second"]]],"name":"series.12345.my_type.0_first.123","columns":["time","Type","Payload","Hostname","Pid","Logger","Severity","EnvVersion","intField","strField","strField2"]}]`
c.Expect(string(result), gs.Equals, expected)
})
c.Specify("skips specified correctly", func() {
conf.Config["skip_fields"] = "Payload strField Type"
err = encoder.Init(conf)
c.Assume(err, gs.IsNil)
result, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `[{"points":[[54321000,"hostname",12345,"Logger",4,"",[123,456]]],"name":"series","columns":["time","Hostname","Pid","Logger","Severity","EnvVersion","intField"]}]`
c.Expect(string(result), gs.Equals, expected)
})
})
}
func DecoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxDecoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
c.Specify("that uses lpeg and inject_message", func() {
dRunner.EXPECT().Name().Return("serialize")
conf.ScriptFilename = "../lua/testsupport/decoder.lua"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("decodes simple messages", func() {
data := "1376389920 debug id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1376389920000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("id")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "2321")
value, ok = pack.Message.GetFieldValue("url")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "example.com")
value, ok = pack.Message.GetFieldValue("item")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "1")
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "1376389920 bogus id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
c.Specify("Preserves data", func() {
conf.ScriptFilename = "../lua/testsupport/serialize.lua"
conf.PreserveData = true
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
decoder.Shutdown()
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("that only uses write_message", func() {
conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
dRunner.EXPECT().Name().Return("write_message")
err := decoder.Init(conf)
decoder.SetDecoderRunner(dRunner)
c.Assume(err, gs.IsNil)
c.Specify("adds a string field to the message", func() {
data := "string field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(string), gs.Equals, "foo")
})
c.Specify("adds a numeric field to the message", func() {
data := "num field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(float64), gs.Equals, float64(1))
})
c.Specify("adds a boolean field to the message", func() {
data := "bool field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(bool), gs.Equals, true)
})
c.Specify("sets type and payload", func() {
data := "set type and payload"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
})
c.Specify("sets field value with representation", func() {
data := "set field value with representation"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("rep")
c.Expect(len(fields), gs.Equals, 1)
field := fields[0]
values := field.GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "foo")
c.Expect(field.GetRepresentation(), gs.Equals, "representation")
})
c.Specify("sets multiple field string values", func() {
data := "set multiple field string values"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("multi")
c.Expect(len(fields), gs.Equals, 2)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "first")
values = fields[1].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "second")
})
c.Specify("sets field string array value", func() {
data := "set field string array value"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("array")
c.Expect(len(fields), gs.Equals, 1)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 2)
c.Expect(values[0], gs.Equals, "first")
c.Expect(values[1], gs.Equals, "second")
})
})
})
c.Specify("A Multipack SandboxDecoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
supply := make(chan *pipeline.PipelinePack, 3)
pack := pipeline.NewPipelinePack(supply)
pack.Message = getTestMessage()
pack1 := pipeline.NewPipelinePack(supply)
pack2 := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
c.Specify("decodes into multiple packs", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
gomock.InOrder(
dRunner.EXPECT().NewPack().Return(pack1),
dRunner.EXPECT().NewPack().Return(pack2),
)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 3)
c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")
for i := 0; i < 1; i++ {
c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))
}
decoder.Shutdown()
})
})
c.Specify("Linux Cpu Stats decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/linux_loadavg.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes a message", func() {
payload := "0.00 0.01 0.05 3/153 660\n"
pack.Message.SetPayload(payload)
f, err := message.NewField("FilePath", "/proc/loadavg", "")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("1MinAvg")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 0.00)
value, ok = pack.Message.GetFieldValue("5MinAvg")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 0.01)
value, ok = pack.Message.GetFieldValue("15MinAvg")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 0.05)
value, ok = pack.Message.GetFieldValue("NumProcesses")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(3))
value, ok = pack.Message.GetFieldValue("FilePath")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, "/proc/loadavg")
})
c.Specify("decodes an invalid message", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Linux Mem Stats decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/linux_memstats.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes a message", func() {
payload := `MemTotal: 4047616 kB
MemFree: 3135780 kB
HugePages_Free: 0
`
pack.Message.SetPayload(payload)
f, err := message.NewField("FilePath", "/proc/meminfo", "")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("MemTotal")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 4.047616e+06)
value, ok = pack.Message.GetFieldValue("MemFree")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 3.13578e+06)
value, ok = pack.Message.GetFieldValue("HugePages_Free")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("FilePath")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, "/proc/meminfo")
})
c.Specify("decodes an invalid message", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Linux Disk Stats decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/linux_diskstats.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes a message", func() {
payload := " 13903 11393 969224 49444 10780 10161 1511920 4104 0 5064 53468\n"
pack.Message.SetPayload(payload)
f, err := message.NewField("FilePath", "/sys/block/sda/stat", "")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("TickerInterval", int64(2), "")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
// These are in the same order the payload should be
value, ok = pack.Message.GetFieldValue("ReadsCompleted")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(13903))
value, ok = pack.Message.GetFieldValue("ReadsMerged")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(11393))
value, ok = pack.Message.GetFieldValue("SectorsRead")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(969224))
value, ok = pack.Message.GetFieldValue("TimeReading")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(49444))
value, ok = pack.Message.GetFieldValue("WritesCompleted")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(10780))
value, ok = pack.Message.GetFieldValue("WritesMerged")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(10161))
value, ok = pack.Message.GetFieldValue("SectorsWritten")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(1511920))
value, ok = pack.Message.GetFieldValue("TimeWriting")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(4104))
value, ok = pack.Message.GetFieldValue("NumIOInProgress")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("TimeDoingIO")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(5064))
value, ok = pack.Message.GetFieldValue("WeightedTimeDoingIO")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(53468))
value, ok = pack.Message.GetFieldValue("TickerInterval")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(2))
value, ok = pack.Message.GetFieldValue("FilePath")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, "/sys/block/sda/stat")
})
c.Specify("decodes a message with no leading space", func() {
payload := "19092852 0 510563170 15817012 46452019 0 1546950712 262535124 0 23823976 278362684\n"
pack.Message.SetPayload(payload)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
value, ok := pack.Message.GetFieldValue("ReadsCompleted")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(19092852))
})
c.Specify("decodes an invalid message", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Nginx access log decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Apache access log decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/apache_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("rsyslog decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["type"] = "MyTestFormat"
conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
conf.Config["tz"] = "America/Los_Angeles"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n"
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
// Syslog timestamp doesn't support year, so we have to calculate
// it for the current year or else this test will fail every
// January.
year := time.Now().Year()
tStr := fmt.Sprintf("%d Feb 10 12:58:58 -0800", year)
t, err := time.Parse("2006 Jan 02 15:04:05 -0700", tStr)
c.Assume(err, gs.IsNil)
unixT := t.UnixNano()
c.Expect(pack.Message.GetTimestamp(), gs.Equals, unixT)
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322))
c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.")
c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"])
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("programname")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "widget")
value, ok = pack.Message.GetFieldValue("syslogfacility")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(3))
value, ok = pack.Message.GetFieldValue("timegenerated")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(1392065939000000000))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("mysql decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["truncate_sql"] = int64(5)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decode standard slow query log", func() {
data := `# User@Host: syncrw[syncrw] @ [127.0.0.1]
# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773
use widget;
SET last_insert_id=999,insert_id=1000,timestamp=1399500744;
# administrator command: do something
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;`
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1399500744000000000))
c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query")
decoder.Shutdown()
})
})
c.Specify("mariadb decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["truncate_sql"] = int64(5)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decode standard slow query log", func() {
data := `# User@Host: syncrw[syncrw] @ [127.0.0.1]
# Thread_id: 110804 Schema: weave0 QC_hit: No
# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;`
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1399500744000000000))
c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query")
decoder.Shutdown()
})
})
}
func PayloadEncoderSpec(c gs.Context) {
c.Specify("A PayloadEncoder", func() {
encoder := new(PayloadEncoder)
config := encoder.ConfigStruct().(*PayloadEncoderConfig)
tsFormat := config.TsFormat
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
payload := "This is the payload!"
pack.Message.SetPayload(payload)
ts := time.Now()
pack.Message.SetTimestamp(ts.UnixNano())
var (
output []byte
err error
)
c.Specify("works with default config options", func() {
err = encoder.Init(config)
c.Expect(err, gs.IsNil)
output, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
c.Expect(string(output), gs.Equals, fmt.Sprintln(payload))
})
c.Specify("honors append_newlines = false", func() {
config.AppendNewlines = false
err = encoder.Init(config)
c.Expect(err, gs.IsNil)
output, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
c.Expect(string(output), gs.Equals, payload)
})
c.Specify("prefixes timestamp", func() {
config.PrefixTs = true
err = encoder.Init(config)
c.Expect(err, gs.IsNil)
output, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
formattedTime := ts.Format(tsFormat)
expected := fmt.Sprintf("%s %s\n", formattedTime, payload)
c.Expect(string(output), gs.Equals, expected)
})
c.Specify("supports alternate time format", func() {
config.PrefixTs = true
tsFormat = time.RFC1123
config.TsFormat = tsFormat
err = encoder.Init(config)
c.Expect(err, gs.IsNil)
output, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
formattedTime := ts.Format(tsFormat)
expected := fmt.Sprintf("%s %s\n", formattedTime, payload)
c.Expect(string(output), gs.Equals, expected)
})
})
}
func FilterSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
fth := NewFilterTestHelper(ctrl)
inChan := make(chan *pipeline.PipelinePack, 1)
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxFilter", func() {
sbFilter := new(SandboxFilter)
config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
config.ScriptType = "lua"
config.MemoryLimit = 32000
config.InstructionLimit = 1000
config.OutputLimit = 1024
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("Over inject messages from ProcessMessage", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(3)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(2)
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("exceeded InjectMessage count"))
config.ScriptFilename = "../lua/testsupport/processinject.lua"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
close(inChan)
sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
})
c.Specify("Over inject messages from TimerEvent", func() {
var timer <-chan time.Time
timer = time.Tick(time.Duration(1) * time.Millisecond)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(12)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(11)
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("exceeded InjectMessage count"))
config.ScriptFilename = "../lua/testsupport/timerinject.lua"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
go func() {
time.Sleep(time.Duration(250) * time.Millisecond)
close(inChan)
}()
sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
})
c.Specify("Preserves data", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
config.ScriptFilename = "../lua/testsupport/serialize.lua"
config.PreserveData = true
sbFilter.SetName("serialize")
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("A SandboxManagerFilter", func() {
sbmFilter := new(SandboxManagerFilter)
config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
config.MaxFilters = 1
origBaseDir := pipeline.Globals().BaseDir
pipeline.Globals().BaseDir = os.TempDir()
sbxMgrsDir := filepath.Join(pipeline.Globals().BaseDir, "sbxmgrs")
defer func() {
pipeline.Globals().BaseDir = origBaseDir
tmpErr := os.RemoveAll(sbxMgrsDir)
c.Expect(tmpErr, gs.IsNil)
}()
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("Control message in the past", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: 5 seconds skew"))
inChan <- pack
close(inChan)
sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
})
c.Specify("Control message in the future", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: -5 seconds skew"))
inChan <- pack
close(inChan)
sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
})
c.Specify("Generates the right default working directory", func() {
sbmFilter.Init(config)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
name := "SandboxManagerFilter"
fth.MockFilterRunner.EXPECT().Name().Return(name)
close(inChan)
sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
_, err := os.Stat(sbxMgrsDir)
c.Expect(err, gs.IsNil)
})
c.Specify("Sanity check the default sandbox configuration limits", func() {
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
})
c.Specify("Sanity check the user specified sandbox configuration limits", func() {
config.MemoryLimit = 123456
config.InstructionLimit = 4321
config.OutputLimit = 8765
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
})
})
}
func NsqInputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
var wg sync.WaitGroup
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
defer close(errChan)
defer close(retPackChan)
c.Specify("A nsq input", func() {
input := new(NsqInput)
ith := new(plugins_ts.InputTestHelper)
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
config := input.ConfigStruct().(*NsqInputConfig)
config.Topic = "test_topic"
config.Channel = "test_channel"
startInput := func() {
wg.Add(1)
go func() {
errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
wg.Done()
}()
}
var mockConsumer *MockConsumer
input.newConsumer = func(topic, channel string, config *nsq.Config) (Consumer, error) {
mockConsumer, _ = NewMockConsumer(topic, channel, config)
return mockConsumer, nil
}
ith.Pack = pipeline.NewPipelinePack(pConfig.InputRecycleChan())
ith.PackSupply = make(chan *pipeline.PipelinePack, 1)
ith.PackSupply <- ith.Pack
inputName := "NsqInput"
ith.MockInputRunner.EXPECT().Name().Return(inputName).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
c.Specify("that is started", func() {
input.SetPipelineConfig(pConfig)
c.Specify("gets messages its subscribed to", func() {
c.Specify("injects messages into the pipeline when not configured with a decoder", func() {
ith.MockInputRunner.EXPECT().Inject(ith.Pack).Do(func(p *pipeline.PipelinePack) {
retPackChan <- p
}).AnyTimes()
err := input.Init(config)
c.Expect(err, gs.IsNil)
c.Expect(input.DecoderName, gs.Equals, "")
})
c.Specify("injects messages into the decoder when configured", func() {
decoderName := "ScribbleDecoder"
config.DecoderName = decoderName
decoder := new(plugins.ScribbleDecoder)
decoder.Init(&plugins.ScribbleDecoderConfig{})
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
mockDecoderRunner.EXPECT().InChan().Return(retPackChan)
ith.MockHelper.EXPECT().DecoderRunner(decoderName,
fmt.Sprintf("%s-%s", inputName, decoderName),
).Return(mockDecoderRunner, true)
err := input.Init(config)
c.Expect(err, gs.IsNil)
c.Expect(input.DecoderName, gs.Equals, decoderName)
})
startInput()
// Should get two finished conn calls since we connect
// to lookupds and nsqds
c.Expect(<-mockConsumer.finishedConn, gs.IsTrue)
c.Expect(<-mockConsumer.finishedConn, gs.IsTrue)
id := nsq.MessageID{}
body := []byte("test")
msg := nsq.NewMessage(id, body)
for _, handler := range mockConsumer.handlers {
handler.HandleMessage(msg)
}
p := <-retPackChan
c.Expect(p.Message.GetPayload(), gs.Equals, "test")
})
input.Stop()
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
})
})
}