func TestLoadDir(t *testing.T) {
origGlobals := pipeline.Globals
origAvailablePlugins := make(map[string]func() interface{})
for k, v := range pipeline.AvailablePlugins {
origAvailablePlugins[k] = v
}
defer func() {
pipeline.Globals = origGlobals
pipeline.AvailablePlugins = origAvailablePlugins
}()
pipeConfig := pipeline.NewPipelineConfig(nil)
confDirPath := "../../plugins/testsupport/config_dir"
err := loadFullConfig(pipeConfig, &confDirPath)
if err != nil {
t.Fatal(err)
}
// verify the inputs sections load properly with a custom name
udp, ok := pipeConfig.InputRunners["UdpInput"]
if !ok {
t.Fatal("No UdpInput configured.")
}
defer udp.Input().Stop()
// and the decoders sections load
_, ok = pipeConfig.DecoderWrappers["ProtobufDecoder"]
if !ok {
t.Fatal("No ProtobufDecoder configured.")
}
// and the outputs sections load
_, ok = pipeConfig.OutputRunners["LogOutput"]
if !ok {
t.Fatal("No LogOutput configured")
}
// and the filters sections load
_, ok = pipeConfig.FilterRunners["sample"]
if !ok {
t.Fatal("No `sample` filter configured.")
}
// and the encoders sections load
encoder, ok := pipeConfig.Encoder("PayloadEncoder", "foo")
if !ok {
t.Fatal("No PayloadEncoder configured.")
}
_, ok = encoder.(*plugins.PayloadEncoder)
if !ok {
t.Fatal("PayloadEncoder isn't a PayloadEncoder")
}
// and that the non "*.toml" file did *not* load
_, ok = pipeConfig.FilterRunners["not_loaded"]
if ok {
t.Fatal("`not_loaded` filter *was* loaded, shouldn't have been!")
}
}
func main() {
configFile := flag.String("config", "agent.conf", "Agent Config file")
maxprocs := flag.Int("maxprocs", 1, "Go runtime MAXPROCS value")
poolSize := flag.Int("poolsize", 1000, "Pipeline pool size")
pprofName := flag.String("pprof", "", "Go profiler output file")
version := flag.Bool("version", false, "Output version and exit")
flag.Parse()
if *version {
fmt.Println(VERSION)
os.Exit(0)
}
runtime.GOMAXPROCS(*maxprocs)
if *pprofName != "" {
profFile, err := os.Create(*pprofName)
if err != nil {
log.Fatalln(err)
}
pprof.StartCPUProfile(profFile)
defer pprof.StopCPUProfile()
}
// Set up and load the pipeline configuration and start the daemon.
pipeconf := pipeline.NewPipelineConfig(*poolSize)
err := pipeconf.LoadFromConfigFile(*configFile)
if err != nil {
log.Fatal("Error reading config: ", err)
}
pipeline.Run(pipeconf)
}
func main() {
configFile := flag.String("config", "/etc/hekad.toml", "Config file")
maxprocs := flag.Int("maxprocs", 1, "Go runtime MAXPROCS value")
poolSize := flag.Int("poolsize", 100, "Pipeline pool size")
decoderPoolSize := flag.Int("decoder_poolsize", 4, "Decoder pool size")
chanSize := flag.Int("plugin_chansize", 50, "Plugin input channel buffer size")
cpuProfName := flag.String("cpuprof", "", "Go CPU profiler output file")
memProfName := flag.String("memprof", "", "Go memory profiler output file")
version := flag.Bool("version", false, "Output version and exit")
maxMsgLoops := flag.Uint("max_message_loops", 4, "Maximum number of times a message can pass thru the system")
flag.Parse()
if *version {
fmt.Println(VERSION)
os.Exit(0)
}
runtime.GOMAXPROCS(*maxprocs)
if *cpuProfName != "" {
profFile, err := os.Create(*cpuProfName)
if err != nil {
log.Fatalln(err)
}
pprof.StartCPUProfile(profFile)
defer pprof.StopCPUProfile()
}
if *memProfName != "" {
defer func() {
profFile, err := os.Create(*memProfName)
if err != nil {
log.Fatalln(err)
}
pprof.WriteHeapProfile(profFile)
profFile.Close()
}()
}
// Set up and load the pipeline configuration and start the daemon.
globals := pipeline.DefaultGlobals()
globals.PoolSize = *poolSize
globals.DecoderPoolSize = *decoderPoolSize
globals.PluginChanSize = *chanSize
globals.MaxMsgLoops = *maxMsgLoops
if globals.MaxMsgLoops == 0 {
globals.MaxMsgLoops = 1
}
pipeconf := pipeline.NewPipelineConfig(globals)
err := pipeconf.LoadFromConfigFile(*configFile)
if err != nil {
log.Fatal("Error reading config: ", err)
}
pipeline.Run(pipeconf)
}
func TestCustomHostname(t *testing.T) {
expected := "my.example.com"
configPath := "../../pipeline/testsupport/sample-hostname.toml"
config, err := LoadHekadConfig(configPath)
if err != nil {
t.Fatal(err)
}
if config.Hostname != expected {
t.Fatalf("HekadConfig.Hostname expected: '%s', Got: %s", expected, config.Hostname)
}
globals, _, _ := setGlobalConfigs(config)
if globals.Hostname != expected {
t.Fatalf("globals.Hostname expected: '%s', Got: %s", expected, globals.Hostname)
}
pConfig := pipeline.NewPipelineConfig(globals)
if pConfig.Hostname() != expected {
t.Fatalf("PipelineConfig.Hostname expected: '%s', Got: %s", expected, pConfig.Hostname())
}
err = loadFullConfig(pConfig, &configPath)
if err != nil {
t.Fatalf("Error loading full config: %s", err.Error())
}
}
func main() {
configFile := flag.String("config", "/etc/hekad.toml", "Config file")
version := flag.Bool("version", false, "Output version and exit")
flag.Parse()
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
if *version {
fmt.Println(VERSION)
os.Exit(0)
}
config, err := LoadHekadConfig(*configFile)
if err != nil {
log.Fatal("Error reading config: ", err)
}
maxprocs := config.Maxprocs
poolSize := config.PoolSize
decoderPoolSize := config.DecoderPoolSize
chanSize := config.ChanSize
cpuProfName := config.CpuProfName
memProfName := config.MemProfName
maxMsgLoops := config.MaxMsgLoops
maxMsgProcessInject := config.MaxMsgProcessInject
maxMsgTimerInject := config.MaxMsgTimerInject
runtime.GOMAXPROCS(maxprocs)
if cpuProfName != "" {
profFile, err := os.Create(cpuProfName)
if err != nil {
log.Fatalln(err)
}
profFile.Close()
pprof.StartCPUProfile(profFile)
defer pprof.StopCPUProfile()
}
if memProfName != "" {
defer func() {
profFile, err := os.Create(memProfName)
if err != nil {
log.Fatalln(err)
}
pprof.WriteHeapProfile(profFile)
profFile.Close()
}()
}
// Set up and load the pipeline configuration and start the daemon.
globals := pipeline.DefaultGlobals()
globals.PoolSize = poolSize
globals.DecoderPoolSize = decoderPoolSize
globals.PluginChanSize = chanSize
globals.MaxMsgLoops = maxMsgLoops
if globals.MaxMsgLoops == 0 {
globals.MaxMsgLoops = 1
}
globals.MaxMsgProcessInject = maxMsgProcessInject
globals.MaxMsgTimerInject = maxMsgTimerInject
pipeconf := pipeline.NewPipelineConfig(globals)
err = pipeconf.LoadFromConfigFile(*configFile)
if err != nil {
log.Fatal("Error reading config: ", err)
}
pipeline.Run(pipeconf)
}
func FilterSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
fth := NewFilterTestHelper(ctrl)
inChan := make(chan *pipeline.PipelinePack, 1)
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxFilter", func() {
sbFilter := new(SandboxFilter)
sbFilter.SetPipelineConfig(pConfig)
config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
config.MemoryLimit = 32000
config.InstructionLimit = 1000
config.OutputLimit = 1024
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("Uninitialized", func() {
err := sbFilter.ReportMsg(msg)
c.Expect(err, gs.IsNil)
})
c.Specify("Over inject messages from ProcessMessage", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(2)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(2)
config.ScriptFilename = "../lua/testsupport/processinject.lua"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
termErr := pipeline.TerminatedError("exceeded InjectMessage count")
c.Expect(err.Error(), gs.Equals, termErr.Error())
})
c.Specify("Over inject messages from TimerEvent", func() {
var timer <-chan time.Time
timer = time.Tick(time.Duration(1) * time.Millisecond)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(11)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(11)
config.ScriptFilename = "../lua/testsupport/timerinject.lua"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
go func() {
time.Sleep(time.Duration(250) * time.Millisecond)
close(inChan)
}()
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
termErr := pipeline.TerminatedError("exceeded InjectMessage count")
c.Expect(err.Error(), gs.Equals, termErr.Error())
})
c.Specify("Preserves data", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
config.ScriptFilename = "../lua/testsupport/serialize.lua"
config.PreserveData = true
sbFilter.SetName("serialize")
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("A SandboxManagerFilter", func() {
pConfig.Globals.BaseDir = os.TempDir()
sbxMgrsDir := filepath.Join(pConfig.Globals.BaseDir, "sbxmgrs")
defer func() {
tmpErr := os.RemoveAll(sbxMgrsDir)
c.Expect(tmpErr, gs.IsNil)
}()
sbmFilter := new(SandboxManagerFilter)
sbmFilter.SetPipelineConfig(pConfig)
config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
config.MaxFilters = 1
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("Control message in the past", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: 5 seconds skew"))
inChan <- pack
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
})
c.Specify("Control message in the future", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: -5 seconds skew"))
inChan <- pack
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
})
c.Specify("Generates the right default working directory", func() {
sbmFilter.Init(config)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
name := "SandboxManagerFilter"
fth.MockFilterRunner.EXPECT().Name().Return(name)
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
_, err = os.Stat(sbxMgrsDir)
c.Expect(err, gs.IsNil)
})
c.Specify("Sanity check the default sandbox configuration limits", func() {
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
})
c.Specify("Sanity check the user specified sandbox configuration limits", func() {
config.MemoryLimit = 123456
config.InstructionLimit = 4321
config.OutputLimit = 8765
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
})
c.Specify("Creates a SandboxFilter runner", func() {
sbxName := "SandboxFilter"
sbxMgrName := "SandboxManagerFilter"
code := `
require("cjson")
function process_message()
inject_payload(cjson.encode({a = "b"}))
return 0
end
`
cfg := `
[%s]
type = "SandboxFilter"
message_matcher = "TRUE"
script_type = "lua"
`
cfg = fmt.Sprintf(cfg, sbxName)
msg.SetPayload(code)
f, err := message.NewField("config", cfg, "toml")
c.Assume(err, gs.IsNil)
msg.AddField(f)
fMatchChan := pConfig.Router().AddFilterMatcher()
errChan := make(chan error)
fth.MockFilterRunner.EXPECT().Name().Return(sbxMgrName)
fullSbxName := fmt.Sprintf("%s-%s", sbxMgrName, sbxName)
fth.MockHelper.EXPECT().Filter(fullSbxName).Return(nil, false)
fth.MockFilterRunner.EXPECT().LogMessage(fmt.Sprintf("Loading: %s", fullSbxName))
sbmFilter.Init(config)
go func() {
err := sbmFilter.loadSandbox(fth.MockFilterRunner, fth.MockHelper, sbxMgrsDir,
msg)
errChan <- err
}()
fMatch := <-fMatchChan
c.Expect(fMatch.MatcherSpecification().String(), gs.Equals, "TRUE")
c.Expect(<-errChan, gs.IsNil)
go func() {
<-pConfig.Router().RemoveFilterMatcher()
}()
ok := pConfig.RemoveFilterRunner(fullSbxName)
c.Expect(ok, gs.IsTrue)
})
})
c.Specify("A Cpu Stats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "cpustats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/cpustats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
msg := getTestMessage()
fields := make([]*message.Field, 4)
fields[0], _ = message.NewField("1MinAvg", 0.08, "")
fields[1], _ = message.NewField("5MinAvg", 0.04, "")
fields[2], _ = message.NewField("15MinAvg", 0.02, "")
fields[3], _ = message.NewField("NumProcesses", 5, "")
msg.Fields = fields
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("cpustats")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with cpuload data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
for i := 1; i <= 3; i++ {
// Fill in the data
t := int64(i * 1000000000)
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":1,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"1MinAvg","unit":"Count","aggregation":"max"},{"name":"5MinAvg","unit":"Count","aggregation":"max"},{"name":"15MinAvg","unit":"Count","aggregation":"max"},{"name":"NumProcesses","unit":"Count","aggregation":"max"}]}
0.08 0.04 0.02 5
0.08 0.04 0.02 5
0.08 0.04 0.02 5
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("A Memstats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "memstats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/memstats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
msg := getTestMessage()
field_names := []string{"MemFree", "Cached", "Active", "Inactive", "VmallocUsed", "Shmem", "SwapCached", "SwapTotal", "SwapFree"}
fields := make([]*message.Field, len(field_names))
for i, name := range field_names {
fields[i], _ = message.NewField(name, 100, "")
}
msg.Fields = fields
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("memstats")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with memstats data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
for i := 1; i <= 3; i++ {
// Fill in the data
t := int64(i * 1000000000)
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":1,"rows":3,"columns":9,"seconds_per_row":1,"column_info":[{"name":"MemFree","unit":"Count","aggregation":"max"},{"name":"Cached","unit":"Count","aggregation":"max"},{"name":"Active","unit":"Count","aggregation":"max"},{"name":"Inactive","unit":"Count","aggregation":"max"},{"name":"VmallocUsed","unit":"Count","aggregation":"max"},{"name":"Shmem","unit":"Count","aggregation":"max"},{"name":"SwapCached","unit":"Count","aggregation":"max"},{"name":"SwapFree","unit":"Count","aggregation":"max"},{"name":"SwapUsed","unit":"Count","aggregation":"max"}]}
100 100 100 100 100 100 100 100 0
100 100 100 100 100 100 100 100 0
100 100 100 100 100 100 100 100 0
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("A diskstats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "diskstats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/diskstats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retMsgChan := make(chan *message.Message, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retMsgChan)
}()
msg := getTestMessage()
field_names := []string{
"WritesCompleted",
"ReadsCompleted",
"SectorsWritten",
"SectorsRead",
"WritesMerged",
"ReadsMerged",
"TimeWriting",
"TimeReading",
"TimeDoingIO",
"WeightedTimeDoingIO",
}
num_fields := len(field_names) + 1
fields := make([]*message.Field, num_fields)
msg.Fields = fields
timeInterval, _ := message.NewField("TickerInterval", 1, "")
fields[num_fields-1] = timeInterval
fieldVal := 100
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelineConfig().AnyTimes()
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).AnyTimes()
fth.MockFilterRunner.EXPECT().Ticker().Return(timer).AnyTimes()
fth.MockFilterRunner.EXPECT().InChan().Return(inChan).AnyTimes()
fth.MockFilterRunner.EXPECT().Name().Return("diskstats").AnyTimes()
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
msg := pack.Message
pack.Message = new(message.Message)
retMsgChan <- msg
}).Return(true).AnyTimes()
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with diskstats data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
// Iterate 4 times since the first one doesn't actually set the cbuf
// in order to set the delta in the cbuf
for i := 1; i <= 4; i++ {
// Fill in the fields
for i, name := range field_names {
fields[i], _ = message.NewField(name, fieldVal, "")
}
// Scale up the value so we can see the delta growing
// by 100 each iteration
fieldVal += i * 100
t := int64(i * 1000000000)
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
testExpects := map[string]string{
"Time doing IO": `{"time":2,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"TimeWriting","unit":"ms","aggregation":"max"},{"name":"TimeReading","unit":"ms","aggregation":"max"},{"name":"TimeDoingIO","unit":"ms","aggregation":"max"},{"name":"WeightedTimeDoi","unit":"ms","aggregation":"max"}]}
200 200 200 200
400 400 400 400
700 700 700 700
`,
"Disk Stats": `{"time":2,"rows":3,"columns":6,"seconds_per_row":1,"column_info":[{"name":"WritesCompleted","unit":"per_1_s","aggregation":"none"},{"name":"ReadsCompleted","unit":"per_1_s","aggregation":"none"},{"name":"SectorsWritten","unit":"per_1_s","aggregation":"none"},{"name":"SectorsRead","unit":"per_1_s","aggregation":"none"},{"name":"WritesMerged","unit":"per_1_s","aggregation":"none"},{"name":"ReadsMerged","unit":"per_1_s","aggregation":"none"}]}
100 100 100 100 100 100
200 200 200 200 200 200
300 300 300 300 300 300
`,
}
timer <- time.Now()
for i := 0; i < 2; i++ {
m := <-retMsgChan
name, ok := m.GetFieldValue("payload_name")
c.Assume(ok, gs.IsTrue)
nameVal, ok := name.(string)
c.Assume(ok, gs.IsTrue)
c.Expect(m.GetPayload(), gs.Equals, testExpects[nameVal])
}
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("http_status filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "http_status"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/http_status.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(2)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
field, _ := message.NewField("status", 0, "")
msg := &message.Message{}
msg.SetTimestamp(0)
msg.AddField(field)
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("http_status")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with http status data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
for i := 0; i <= 6; i++ {
msg.Fields[0].ValueInteger[0] = int64(i * 100) // iterate through the status codes with a bogus status on each end
pack.Message = msg
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":0,"rows":2,"columns":6,"seconds_per_row":1,"column_info":[{"name":"HTTP_100","unit":"count","aggregation":"sum"},{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1 1 1 1 1 2
nan nan nan nan nan nan
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
}
func DashboardOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
dashboardOutput := new(DashboardOutput)
dashboardOutput.pConfig = pConfig
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)
errChan := make(chan error, 1)
startOutput := func() {
go func() {
errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
}()
}
if runtime.GOOS != "windows" {
c.Specify("A DashboardOutput", func() {
tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
c.Assume(err, gs.IsNil)
config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
config.WorkingDirectory = tmpdir
c.Specify("Init halts if basedirectory is not writable", func() {
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
err = dashboardOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("that is running", func() {
startedChan := make(chan bool, 1)
defer close(startedChan)
ts := httptest.NewUnstartedServer(nil)
dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
ts.Start()
startedChan <- true
return nil
}
ticker := make(chan time.Time)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)
err := os.MkdirAll(tmpdir, 0700)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Noop
})
c.Specify("sets custom http headers", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err = dashboardOutput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = dashboardOutput.server
startOutput()
inChan <- pack
<-startedChan
resp, err := http.Get(ts.URL)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
// Verify headers are there
eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
c.Expect(eq, gs.IsTrue)
eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
c.Expect(eq, gs.IsTrue)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
ts.Close()
})
})
}
}
func EncoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
// NewPipelineConfig sets up Globals which is needed for the
// pipeline.Prepend*Dir functions to not die during plugin Init().
_ = pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxEncoder", func() {
encoder := new(SandboxEncoder)
conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
pack.Message.SetPayload("original")
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321)
pack.Message.SetUuid(uuid.NewRandom())
var (
result []byte
err error
)
c.Specify("emits JSON correctly", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_json.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = json.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
c.Expect(msg.GetPid(), gs.Equals, int32(12345))
c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
c.Expect(msg.GetHostname(), gs.Equals, "hostname")
c.Expect(msg.GetPayload(), gs.Equals, "original")
c.Expect(msg.GetType(), gs.Equals, "my_type")
})
c.Specify("emits text correctly", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_text.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
c.Expect(string(result), gs.Equals, "Prefixed original")
})
c.Specify("emits protobuf correctly", func() {
c.Specify("when inject_message is used", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_protobuf.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = proto.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
c.Expect(msg.GetPid(), gs.Equals, int32(12345))
c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
c.Expect(msg.GetHostname(), gs.Equals, "hostname")
c.Expect(msg.GetPayload(), gs.Equals, "mutated")
c.Expect(msg.GetType(), gs.Equals, "after")
})
c.Specify("when `write_message` is used", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_writemessage.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = proto.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetPayload(), gs.Equals, "mutated payload")
c.Expect(pack.Message.GetPayload(), gs.Equals, "original")
})
})
})
}
func FilterSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
fth := NewFilterTestHelper(ctrl)
inChan := make(chan *pipeline.PipelinePack, 1)
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxFilter", func() {
sbFilter := new(SandboxFilter)
sbFilter.SetPipelineConfig(pConfig)
config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
config.MemoryLimit = 64000
config.InstructionLimit = 1000
config.OutputLimit = 1024
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
pack.Message = msg
c.Specify("Uninitialized", func() {
err := sbFilter.ReportMsg(msg)
c.Expect(err, gs.IsNil)
})
c.Specify("Over inject messages from ProcessMessage", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(2)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(2)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/processinject.lua"
config.ModuleDirectory = "../lua/modules"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
var termErr pipeline.TerminatedError
if runtime.GOOS == "windows" {
termErr = pipeline.TerminatedError("process_message() ..\\lua\\testsupport\\processinject.lua:8: inject_payload() exceeded InjectMessage count")
} else {
termErr = pipeline.TerminatedError("process_message() ../lua/testsupport/processinject.lua:8: inject_payload() exceeded InjectMessage count")
}
c.Expect(err.Error(), gs.Equals, termErr.Error())
})
c.Specify("Over inject messages from TimerEvent", func() {
var timer <-chan time.Time
timer = time.Tick(time.Duration(1) * time.Millisecond)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(11)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(11)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/timerinject.lua"
config.ModuleDirectory = "../lua/modules"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
go func() {
time.Sleep(time.Duration(250) * time.Millisecond)
close(inChan)
}()
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
var termErr pipeline.TerminatedError
if runtime.GOOS == "windows" {
termErr = pipeline.TerminatedError("timer_event() ..\\lua\\testsupport\\timerinject.lua:13: inject_payload() exceeded InjectMessage count")
} else {
termErr = pipeline.TerminatedError("timer_event() ../lua/testsupport/timerinject.lua:13: inject_payload() exceeded InjectMessage count")
}
c.Expect(err.Error(), gs.Equals, termErr.Error())
})
c.Specify("Preserves data", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/serialize.lua"
config.ModuleDirectory = "../lua/modules"
config.PreserveData = true
sbFilter.SetName("serialize")
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
c.Specify("process_message error string", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("script provided error message"))
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/process_message_error_string.lua"
config.ModuleDirectory = "../lua/modules"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
})
c.Specify("Over inject messages from the shutdown TimerEvent", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(10)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(10)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(10)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
config.ScriptFilename = "../lua/testsupport/timerinject.lua"
config.ModuleDirectory = "../lua/modules"
config.TimerEventOnShutdown = true
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
if runtime.GOOS == "windows" {
c.Expect(err.Error(), gs.Equals, "FATAL: timer_event() ..\\lua\\testsupport\\timerinject.lua:13: inject_payload() exceeded InjectMessage count")
} else {
c.Expect(err.Error(), gs.Equals, "FATAL: timer_event() ../lua/testsupport/timerinject.lua:13: inject_payload() exceeded InjectMessage count")
}
})
})
c.Specify("A SandboxManagerFilter", func() {
pConfig.Globals.BaseDir = os.TempDir()
sbxMgrsDir := filepath.Join(pConfig.Globals.BaseDir, "sbxmgrs")
defer func() {
tmpErr := os.RemoveAll(sbxMgrsDir)
c.Expect(tmpErr, gs.IsNil)
}()
sbmFilter := new(SandboxManagerFilter)
sbmFilter.SetPipelineConfig(pConfig)
config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
config.MaxFilters = 1
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
c.Specify("Control message in the past", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UpdateCursor("")
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
pack.BufferedPack = true
pack.DelivErrChan = make(chan error, 1)
inChan <- pack
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
err = <-pack.DelivErrChan
c.Expect(err.Error(), gs.Equals, "Discarded control message: 5 seconds skew")
})
c.Specify("Control message in the future", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UpdateCursor("")
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
pack.BufferedPack = true
pack.DelivErrChan = make(chan error, 1)
inChan <- pack
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
err = <-pack.DelivErrChan
c.Expect(err.Error(), gs.Equals, "Discarded control message: -5 seconds skew")
})
c.Specify("Generates the right default working directory", func() {
sbmFilter.Init(config)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
name := "SandboxManagerFilter"
fth.MockFilterRunner.EXPECT().Name().Return(name)
close(inChan)
err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(err, gs.IsNil)
c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
_, err = os.Stat(sbxMgrsDir)
c.Expect(err, gs.IsNil)
})
c.Specify("Sanity check the default sandbox configuration limits", func() {
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
})
c.Specify("Sanity check the user specified sandbox configuration limits", func() {
config.MemoryLimit = 123456
config.InstructionLimit = 4321
config.OutputLimit = 8765
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
})
c.Specify("Creates a SandboxFilter runner", func() {
sbxName := "SandboxFilter"
sbxMgrName := "SandboxManagerFilter"
code := `
function process_message()
inject_payload('{a = "b"}')
return 0
end
`
cfg := `
[%s]
type = "SandboxFilter"
message_matcher = "TRUE"
script_type = "lua"
`
cfg = fmt.Sprintf(cfg, sbxName)
msg.SetPayload(code)
f, err := message.NewField("config", cfg, "toml")
c.Assume(err, gs.IsNil)
msg.AddField(f)
fMatchChan := pConfig.Router().AddFilterMatcher()
errChan := make(chan error)
fth.MockFilterRunner.EXPECT().Name().Return(sbxMgrName)
fullSbxName := fmt.Sprintf("%s-%s", sbxMgrName, sbxName)
fth.MockHelper.EXPECT().Filter(fullSbxName).Return(nil, false)
fth.MockFilterRunner.EXPECT().LogMessage(fmt.Sprintf("Loading: %s", fullSbxName))
sbmFilter.Init(config)
go func() {
err := sbmFilter.loadSandbox(fth.MockFilterRunner, fth.MockHelper, sbxMgrsDir,
msg)
errChan <- err
}()
fMatch := <-fMatchChan
c.Expect(fMatch.MatcherSpecification().String(), gs.Equals, "TRUE")
c.Expect(<-errChan, gs.IsNil)
go func() {
<-pConfig.Router().RemoveFilterMatcher()
}()
ok := pConfig.RemoveFilterRunner(fullSbxName)
c.Expect(ok, gs.IsTrue)
})
})
c.Specify("A Load Average Stats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "loadavg"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/loadavg.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
msg := getTestMessage()
fields := make([]*message.Field, 4)
fields[0], _ = message.NewField("1MinAvg", 0.08, "")
fields[1], _ = message.NewField("5MinAvg", 0.04, "")
fields[2], _ = message.NewField("15MinAvg", 0.02, "")
fields[3], _ = message.NewField("NumProcesses", 5, "")
msg.Fields = fields
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return("loadavg")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with loadavg data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
var i int64
for i = 1; i <= 3; i++ {
// Fill in the data
t := i * 1000000000
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":1,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"1MinAvg","unit":"Count","aggregation":"max"},{"name":"5MinAvg","unit":"Count","aggregation":"max"},{"name":"15MinAvg","unit":"Count","aggregation":"max"},{"name":"NumProcesses","unit":"Count","aggregation":"max"}]}
0.08 0.04 0.02 5
0.08 0.04 0.02 5
0.08 0.04 0.02 5
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("A Memstats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "memstats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/memstats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
msg := getTestMessage()
field_names := []string{"MemFree", "Cached", "Active", "Inactive", "VmallocUsed", "Shmem", "SwapCached", "SwapTotal", "SwapFree"}
fields := make([]*message.Field, len(field_names))
for i, name := range field_names {
fields[i], _ = message.NewField(name, 100, "")
}
msg.Fields = fields
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return("memstats")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with memstats data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
var i int64
for i = 1; i <= 3; i++ {
// Fill in the data
t := i * 1000000000
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":1,"rows":3,"columns":9,"seconds_per_row":1,"column_info":[{"name":"MemFree","unit":"Count","aggregation":"max"},{"name":"Cached","unit":"Count","aggregation":"max"},{"name":"Active","unit":"Count","aggregation":"max"},{"name":"Inactive","unit":"Count","aggregation":"max"},{"name":"VmallocUsed","unit":"Count","aggregation":"max"},{"name":"Shmem","unit":"Count","aggregation":"max"},{"name":"SwapCached","unit":"Count","aggregation":"max"},{"name":"SwapFree","unit":"Count","aggregation":"max"},{"name":"SwapUsed","unit":"Count","aggregation":"max"}]}
100 100 100 100 100 100 100 100 0
100 100 100 100 100 100 100 100 0
100 100 100 100 100 100 100 100 0
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("A diskstats filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "diskstats"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/diskstats.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(3)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retMsgChan := make(chan *message.Message, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retMsgChan)
}()
msg := getTestMessage()
field_names := []string{
"WritesCompleted",
"ReadsCompleted",
"SectorsWritten",
"SectorsRead",
"WritesMerged",
"ReadsMerged",
"TimeWriting",
"TimeReading",
"TimeDoingIO",
"WeightedTimeDoingIO",
}
num_fields := len(field_names) + 1
fields := make([]*message.Field, num_fields)
msg.Fields = fields
timeInterval, _ := message.NewField("TickerInterval", 1, "")
fields[num_fields-1] = timeInterval
var fieldVal int64 = 100
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelineConfig().AnyTimes()
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).AnyTimes()
fth.MockFilterRunner.EXPECT().Ticker().Return(timer).AnyTimes()
fth.MockFilterRunner.EXPECT().InChan().Return(inChan).AnyTimes()
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return("diskstats").AnyTimes()
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
msg := pack.Message
pack.Message = new(message.Message)
retMsgChan <- msg
}).Return(true).AnyTimes()
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with diskstats data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
// Iterate 4 times since the first one doesn't actually set the cbuf
// in order to set the delta in the cbuf
var i int64
for i = 1; i <= 4; i++ {
// Fill in the fields
for i, name := range field_names {
fields[i], _ = message.NewField(name, fieldVal, "")
}
// Scale up the value so we can see the delta growing
// by 100 each iteration
fieldVal += i * 100
t := i * 1000000000
pack.Message = msg
pack.Message.SetTimestamp(t)
// Feed in a pack
inChan <- pack
pack = <-recycleChan
}
testExpects := map[string]string{
"Time doing IO": `{"time":2,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"TimeWriting","unit":"ms","aggregation":"max"},{"name":"TimeReading","unit":"ms","aggregation":"max"},{"name":"TimeDoingIO","unit":"ms","aggregation":"max"},{"name":"WeightedTimeDoi","unit":"ms","aggregation":"max"}]}
200 200 200 200
400 400 400 400
700 700 700 700
`,
"Disk Stats": `{"time":2,"rows":3,"columns":6,"seconds_per_row":1,"column_info":[{"name":"WritesCompleted","unit":"per_1_s","aggregation":"none"},{"name":"ReadsCompleted","unit":"per_1_s","aggregation":"none"},{"name":"SectorsWritten","unit":"per_1_s","aggregation":"none"},{"name":"SectorsRead","unit":"per_1_s","aggregation":"none"},{"name":"WritesMerged","unit":"per_1_s","aggregation":"none"},{"name":"ReadsMerged","unit":"per_1_s","aggregation":"none"}]}
100 100 100 100 100 100
200 200 200 200 200 200
300 300 300 300 300 300
`,
}
timer <- time.Now()
for i := 0; i < 2; i++ {
m := <-retMsgChan
name, ok := m.GetFieldValue("payload_name")
c.Assume(ok, gs.IsTrue)
nameVal, ok := name.(string)
c.Assume(ok, gs.IsTrue)
c.Expect(m.GetPayload(), gs.Equals, testExpects[nameVal])
}
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("http_status filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "http_status"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/filters/http_status.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
conf.Config["rows"] = int64(2)
conf.Config["sec_per_row"] = int64(1)
timer := make(chan time.Time, 1)
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retPackChan)
}()
field, _ := message.NewField("status", 0, "")
msg := &message.Message{}
msg.SetTimestamp(0)
msg.AddField(field)
pack := pipeline.NewPipelinePack(recycleChan)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return("http_status")
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retPackChan <- pack
}).Return(true)
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("should fill a cbuf with http status data", func() {
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
for i := 0; i <= 6; i++ {
msg.Fields[0].ValueInteger[0] = int64(i * 100) // iterate through the status codes with a bogus status on each end
pack.Message = msg
inChan <- pack
pack = <-recycleChan
}
timer <- time.Now()
p := <-retPackChan
// Check the result of the filter's inject
pl := `{"time":0,"rows":2,"columns":6,"seconds_per_row":1,"column_info":[{"name":"HTTP_100","unit":"count","aggregation":"sum"},{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1 1 1 1 1 2
nan nan nan nan nan nan
`
c.Expect(p.Message.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
c.Specify("influx_batch filter", func() {
filter := new(SandboxFilter)
filter.SetPipelineConfig(pConfig)
filter.name = "influx_batch"
conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
errChan := make(chan error, 1)
retMsgChan := make(chan *message.Message, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
defer func() {
close(errChan)
close(retMsgChan)
}()
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = &message.Message{}
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321 * 1e9)
pack.Message.SetLogger("Logger")
pack.Message.SetPayload("Payload value lorem ipsum")
pack.Message.SetUuid(uuid.Parse("8e414f01-9d7f-4a48-a5e1-ae92e5954df5"))
f, err := message.NewField("intField", 123, "") // Overwritten in a loop below.
c.Assume(err, gs.IsNil)
err = f.AddValue(456)
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("strField", "0_first", "")
c.Assume(err, gs.IsNil)
err = f.AddValue("0_second")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("strField", "1_first", "")
c.Assume(err, gs.IsNil)
err = f.AddValue("1_second")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("byteField", []byte("first"), "")
c.Assume(err, gs.IsNil)
err = f.AddValue([]byte("second"))
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
fth.MockFilterRunner.EXPECT().Ticker().Return(nil)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
fth.MockFilterRunner.EXPECT().Name().Return(filter.name)
fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
retMsgChan <- pack.Message
}).Return(true)
c.Assume(err, gs.IsNil)
conf.ScriptFilename = "../lua/filters/influx_batch.lua"
conf.ModuleDirectory = "../lua/modules"
conf.Config = make(map[string]interface{})
c.Specify("encodes a basic message", func() {
conf.Config["flush_count"] = "6"
err := filter.Init(conf)
c.Assume(err, gs.IsNil)
go func() {
errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
}()
// Having our own reference to the message struct lets us hand it
// back to the pack even after recycling.
msg := pack.Message
for i := 0; i < 6; i++ {
msg.Fields[0].ValueInteger[0] = int64(i * 100)
pack.Message = msg
pack.EncodeMsgBytes()
inChan <- pack
pack = <-recycleChan
}
// timer <- time.Now()
m := <-retMsgChan
// Check the result of the filter's inject
msgStr := `byteField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="first" 54321000
byteField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="second" 54321000
strField_fidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="1_first" 54321000
strField_fidx_1_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="1_second" 54321000
strField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="0_first" 54321000
strField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="0_second" 54321000
intField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value=%d.000000 54321000
intField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value=456.000000 54321000
`
pl := ""
for i := 0; i < 6; i++ {
pl += fmt.Sprintf(msgStr, i*100)
}
c.Expect(m.GetPayload(), gs.Equals, pl)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
})
}
func DecoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
// NewPipelineConfig sets up Globals which is needed for the
// pipeline.Prepend*Dir functions to not die during plugin Init().
_ = pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
conf.ScriptType = "lua"
c.Specify("that uses lpeg and inject_message", func() {
dRunner.EXPECT().Name().Return("serialize")
conf.ScriptFilename = "../lua/testsupport/decoder.lua"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("decodes simple messages", func() {
data := "1376389920 debug id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1376389920000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("id")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "2321")
value, ok = pack.Message.GetFieldValue("url")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "example.com")
value, ok = pack.Message.GetFieldValue("item")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "1")
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "1376389920 bogus id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
c.Specify("Preserves data", func() {
conf.ScriptFilename = "../lua/testsupport/serialize.lua"
conf.PreserveData = true
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
decoder.Shutdown()
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("that only uses write_message", func() {
conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
dRunner.EXPECT().Name().Return("write_message")
err := decoder.Init(conf)
decoder.SetDecoderRunner(dRunner)
c.Assume(err, gs.IsNil)
c.Specify("adds a string field to the message", func() {
data := "string field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(string), gs.Equals, "foo")
})
c.Specify("adds a numeric field to the message", func() {
data := "num field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(float64), gs.Equals, float64(1))
})
c.Specify("adds a boolean field to the message", func() {
data := "bool field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(bool), gs.Equals, true)
})
c.Specify("sets type and payload", func() {
data := "set type and payload"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
})
c.Specify("sets field value with representation", func() {
data := "set field value with representation"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("rep")
c.Expect(len(fields), gs.Equals, 1)
field := fields[0]
values := field.GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "foo")
c.Expect(field.GetRepresentation(), gs.Equals, "representation")
})
c.Specify("sets multiple field string values", func() {
data := "set multiple field string values"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("multi")
c.Expect(len(fields), gs.Equals, 2)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "first")
values = fields[1].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "second")
})
c.Specify("sets field string array value", func() {
data := "set field string array value"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("array")
c.Expect(len(fields), gs.Equals, 1)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 2)
c.Expect(values[0], gs.Equals, "first")
c.Expect(values[1], gs.Equals, "second")
})
})
})
c.Specify("A Multipack SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
conf.ScriptType = "lua"
supply := make(chan *pipeline.PipelinePack, 3)
pack := pipeline.NewPipelinePack(supply)
pack.Message = getTestMessage()
pack1 := pipeline.NewPipelinePack(supply)
pack2 := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
c.Specify("decodes into multiple packs", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
gomock.InOrder(
dRunner.EXPECT().NewPack().Return(pack1),
dRunner.EXPECT().NewPack().Return(pack2),
)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 3)
c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")
for i := 0; i < 1; i++ {
c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))
}
decoder.Shutdown()
})
})
c.Specify("Nginx access log decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.ScriptType = "lua"
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("rsyslog decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.ScriptType = "lua"
conf.Config = make(map[string]interface{})
conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
conf.Config["tz"] = "America/Los_Angeles"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost kernel: imklog 5.8.6, log source = /proc/kmsg started.\n"
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392065938000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
c.Expect(pack.Message.GetPayload(), gs.Equals, "imklog 5.8.6, log source = /proc/kmsg started.")
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("syslogtag")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "kernel:")
value, ok = pack.Message.GetFieldValue("syslogfacility")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(3))
value, ok = pack.Message.GetFieldValue("timegenerated")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(1392065939000000000))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
}
func HttpInputSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
jsonPost := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`
c.Specify("A HttpInput", func() {
httpInput := pipeline.HttpInput{}
c.Specify("honors time ticker to flush", func() {
ith := new(pipeline.InputTestHelper)
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
startInput := func() {
go func() {
err := httpInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
}
ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
ith.Decoders = make([]DecoderRunner, int(message.Header_JSON+1))
ith.Decoders[message.Header_JSON] = NewMockDecoderRunner(ctrl)
ith.MockDecoderSet = NewMockDecoderSet(ctrl)
// Spin up a http server
server, err := ts.NewOneHttpServer(jsonPost, "localhost", 9876)
c.Expect(err, gs.IsNil)
go server.Start("/")
time.Sleep(10 * time.Millisecond)
config := httpInput.ConfigStruct().(*HttpInputConfig)
config.DecoderName = "JsonDecoder"
config.Url = "http://localhost:9876/"
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
mockDecoderRunner := ith.Decoders[message.Header_JSON].(*MockDecoderRunner)
// Stub out the DecoderRunner input channel so that we can
// inspect bytes later on
dRunnerInChan := make(chan *PipelinePack, 1)
mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)
dset := ith.MockDecoderSet.EXPECT().ByName("JsonDecoder")
dset.Return(ith.Decoders[message.Header_JSON], true)
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// We need for the pipeline to finish up
time.Sleep(50 * time.Millisecond)
})
c.Specify("short circuits packs into the router", func() {
ith := new(InputTestHelper)
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
startInput := func() {
go func() {
err := httpInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
}
ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
ith.Decoders = make([]DecoderRunner, int(message.Header_JSON+1))
ith.Decoders[message.Header_JSON] = NewMockDecoderRunner(ctrl)
ith.MockDecoderSet = NewMockDecoderSet(ctrl)
config := httpInput.ConfigStruct().(*HttpInputConfig)
config.Url = "http://localhost:9876/"
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
err := httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// We need for the pipeline to finish up
time.Sleep(50 * time.Millisecond)
})
})
}
func TestWriteMessage(t *testing.T) {
pipeline.NewPipelineConfig(nil) // Set up globals.
var sbc SandboxConfig
sbc.ScriptFilename = "./testsupport/field_scribble.lua"
sbc.MemoryLimit = 32767
sbc.InstructionLimit = 1000
pack := getTestPack()
sb, err := lua.CreateLuaSandbox(&sbc)
if err != nil {
t.Errorf("%s", err)
}
err = sb.Init("", "decoder")
if err != nil {
t.Errorf("%s", err)
}
r := sb.ProcessMessage(pack)
if r != 0 {
t.Errorf("ProcessMessage should return 0, received %d", r)
}
if pack.Message.GetType() != "MyType" {
t.Error("Type not set")
}
if pack.Message.GetLogger() != "MyLogger" {
t.Error("Logger not set")
}
packTime := time.Unix(0, pack.Message.GetTimestamp())
cmpTime := time.Unix(0, 1385968914904958136)
d, _ := time.ParseDuration("500ns")
packTime = packTime.Round(d)
cmpTime = cmpTime.Round(d)
if !packTime.Equal(cmpTime) {
t.Errorf("Timestamp not set: %d", pack.Message.GetTimestamp())
}
if pack.Message.GetPayload() != "MyPayload" {
t.Error("Payload not set")
}
if pack.Message.GetEnvVersion() != "000" {
t.Error("EnvVersion not set")
}
if pack.Message.GetHostname() != "MyHostname" {
t.Error("Hostname not set")
}
if pack.Message.GetSeverity() != 4 {
t.Error("Severity not set")
}
if pack.Message.GetPid() != 12345 {
t.Error("Pid not set")
}
var f []*message.Field
f = pack.Message.FindAllFields("String")
if len(f) != 1 || len(f[0].GetValueString()) != 1 || f[0].GetValueString()[0] != "foo" {
t.Errorf("String field not set")
}
f = pack.Message.FindAllFields("Float")
if len(f) != 1 || len(f[0].GetValueDouble()) != 1 || f[0].GetValueDouble()[0] != 1.2345 {
t.Error("Float field not set")
}
f = pack.Message.FindAllFields("Int")
if len(f) != 1 {
t.Error("Int field not set")
} else {
if len(f[0].GetValueDouble()) != 2 || f[0].GetValueDouble()[0] != 123 ||
f[0].GetValueDouble()[1] != 456 {
t.Error("Int field set incorrectly")
}
if f[0].GetRepresentation() != "count" {
t.Error("Int field representation not set")
}
}
f = pack.Message.FindAllFields("Bool")
if len(f) != 2 {
t.Error("Bool fields not set")
} else {
if len(f[0].GetValueBool()) != 1 || !f[0].GetValueBool()[0] {
t.Error("Bool field 0 not set")
}
if len(f[1].GetValueBool()) != 1 || f[1].GetValueBool()[0] {
t.Error("Bool field 1 not set")
}
}
if f = pack.Message.FindAllFields(""); len(f) != 1 {
t.Error("No-name field not set")
} else {
if len(f[0].GetValueString()) != 1 || f[0].GetValueString()[0] != "bad idea" {
t.Error("No-name field set incorrectly")
}
}
if pack.Message.GetUuidString() != "550d19b9-58c7-49d8-b0dd-b48cd1c5b305" {
t.Errorf("Uuid not set: %s", pack.Message.GetUuidString())
}
}
func TestWriteMessageErrors(t *testing.T) {
pack := getTestPack()
// NewPipelineConfig sets up Globals for error logging
pipeline.NewPipelineConfig(nil)
tests := []string{
"too few parameters",
"too many parameters",
"Unknown field name",
"Missing fields specifier",
"Missing closing bracket",
"Out of range field index",
"Negative field index",
"Negative array index",
"nil field",
"nil value",
"empty uuid",
"invalid uuid",
"empty timestamp",
"invalid timestamp",
"bool severity",
"double hostname",
}
msgs := []string{
"process_message() ./testsupport/write_message_errors.lua:11: write_message() incorrect number of arguments",
"process_message() ./testsupport/write_message_errors.lua:13: write_message() incorrect number of arguments",
"process_message() ./testsupport/write_message_errors.lua:15: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:17: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:19: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:21: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:23: bad argument #4 to 'write_message' (field index must be >= 0)",
"process_message() ./testsupport/write_message_errors.lua:25: bad argument #5 to 'write_message' (array index must be >= 0)",
"process_message() ./testsupport/write_message_errors.lua:27: bad argument #1 to 'write_message' (string expected, got nil)",
"process_message() ./testsupport/write_message_errors.lua:29: write_message() only accepts numeric, string, or boolean field values",
"process_message() ./testsupport/write_message_errors.lua:31: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:33: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:35: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:37: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:39: write_message() failed",
"process_message() ./testsupport/write_message_errors.lua:41: write_message() failed",
}
var sbc SandboxConfig
sbc.ScriptFilename = "./testsupport/write_message_errors.lua"
sbc.MemoryLimit = 32767
sbc.InstructionLimit = 1000
sbc.OutputLimit = 128
for i, v := range tests {
sb, err := lua.CreateLuaSandbox(&sbc)
if err != nil {
t.Errorf("%s", err)
}
err = sb.Init("", "decoder")
if err != nil {
t.Errorf("%s", err)
}
pack.Message.SetPayload(v)
r := sb.ProcessMessage(pack)
if r != 1 || STATUS_TERMINATED != sb.Status() {
t.Errorf("test: %s status should be %d, received %d",
v, STATUS_TERMINATED, sb.Status())
}
s := sb.LastError()
if s != msgs[i] {
t.Errorf("test: %s error should be \"%s\", received \"%s\"",
v, msgs[i], s)
}
sb.Destroy("")
}
}
func NsqOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
var wg sync.WaitGroup
errChan := make(chan error, 1)
defer close(errChan)
c.Specify("A nsq output", func() {
output := new(NsqOutput)
oth := plugins_ts.NewOutputTestHelper(ctrl)
config := output.ConfigStruct().(*NsqOutputConfig)
config.Topic = "test"
startOutput := func() {
wg.Add(1)
go func() {
errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
}
var mockProducer *MockProducer
output.newProducer = func(addr string, config *nsq.Config) (Producer, error) {
mockProducer, _ = NewMockProducer(addr, config)
return mockProducer, nil
}
msg := pipeline_ts.GetTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
c.Specify("requires at least one address", func() {
err := output.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
config.Addresses = []string{"test.com:4150"}
c.Specify("requires an encoder", func() {
err := output.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(nil)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
encoder := new(plugins.PayloadEncoder)
encoder.Init(&plugins.PayloadEncoderConfig{PrefixTs: false})
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
inChan := make(chan *pipeline.PipelinePack, 1)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
c.Specify("publishes a message with the configured topic", func() {
err := output.Init(config)
c.Assume(err, gs.IsNil)
startOutput()
inChan <- pack
transaction := <-mockProducer.respChan
c.Expect(transaction.Error, gs.IsNil)
close(inChan)
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
})
})
})
}
func OutputSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := pipeline.NewPipelineConfig(nil)
inChan := make(chan *pipeline.PipelinePack, 1)
timer := make(chan time.Time, 1)
c.Specify("A SandboxOutput", func() {
output := new(SandboxOutput)
output.SetPipelineConfig(pConfig)
conf := output.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/output.lua"
conf.ModuleDirectory = "../lua/modules;../lua/testsupport/modules"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
data := "1376389920 debug id=2321 url=example.com item=1"
pack.Message.SetPayload(data)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Ticker().Return(timer)
c.Specify("writes a payload to file", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
inChan <- pack
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Assume(err, gs.IsNil)
tmpFile, err := os.Open("output.lua.txt")
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, data)
c.Expect(output.processMessageCount, gs.Equals, int64(1))
c.Expect(output.processMessageSamples, gs.Equals, int64(1))
c.Expect(output.processMessageFailures, gs.Equals, int64(0))
})
c.Specify("failure processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
pack.Message.SetPayload("FAILURE")
pack.BufferedPack = true
pack.DelivErrChan = make(chan error, 1)
inChan <- pack
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Assume(err, gs.IsNil)
c.Expect(output.processMessageFailures, gs.Equals, int64(1))
err = <-pack.DelivErrChan
c.Expect(err.Error(), gs.Equals, "failure message")
})
c.Specify("user abort processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
e := errors.New("FATAL: user abort")
pack.Message.SetPayload("USERABORT")
inChan <- pack
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err.Error(), gs.Equals, e.Error())
})
c.Specify("fatal error processing data", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
pack.Message.SetPayload("FATAL")
inChan <- pack
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("fatal error in timer_event", func() {
err := output.Init(conf)
c.Expect(err, gs.IsNil)
timer <- time.Now()
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("fatal error in shutdown timer_event", func() {
conf.TimerEventOnShutdown = true
err := output.Init(conf)
c.Expect(err, gs.IsNil)
close(inChan)
err = output.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func main() {
configPath := flag.String("config", filepath.FromSlash("/etc/hekad.toml"),
"Config file or directory. If directory is specified then all files "+
"in the directory will be loaded.")
version := flag.Bool("version", false, "Output version and exit")
flag.Parse()
config := &HekadConfig{}
var err error
var cpuProfName string
var memProfName string
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
if *version {
fmt.Println(VERSION)
os.Exit(0)
}
config, err = LoadHekadConfig(*configPath)
if err != nil {
log.Fatal("Error reading config: ", err)
}
globals, cpuProfName, memProfName := setGlobalConfigs(config)
if err = os.MkdirAll(globals.BaseDir, 0755); err != nil {
log.Fatalf("Error creating base_dir %s: %s", config.BaseDir, err)
}
if cpuProfName != "" {
profFile, err := os.Create(cpuProfName)
if err != nil {
log.Fatalln(err)
}
profFile.Close()
pprof.StartCPUProfile(profFile)
defer pprof.StopCPUProfile()
}
if memProfName != "" {
defer func() {
profFile, err := os.Create(memProfName)
if err != nil {
log.Fatalln(err)
}
pprof.WriteHeapProfile(profFile)
profFile.Close()
}()
}
// Set up and load the pipeline configuration and start the daemon.
pipeconf := pipeline.NewPipelineConfig(globals)
p, err := os.Open(*configPath)
fi, err := p.Stat()
if fi.IsDir() {
files, _ := ioutil.ReadDir(*configPath)
for _, f := range files {
err = pipeconf.LoadFromConfigFile(filepath.Join(*configPath, f.Name()))
}
} else {
err = pipeconf.LoadFromConfigFile(*configPath)
}
if err != nil {
log.Fatal("Error reading config: ", err)
}
pipeline.Run(pipeconf)
}
func DecoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxDecoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
c.Specify("that uses lpeg and inject_message", func() {
dRunner.EXPECT().Name().Return("serialize")
conf.ScriptFilename = "../lua/testsupport/decoder.lua"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("decodes simple messages", func() {
data := "1376389920 debug id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1376389920000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("id")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "2321")
value, ok = pack.Message.GetFieldValue("url")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "example.com")
value, ok = pack.Message.GetFieldValue("item")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "1")
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "1376389920 bogus id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
c.Specify("Preserves data", func() {
conf.ScriptFilename = "../lua/testsupport/serialize.lua"
conf.PreserveData = true
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
decoder.Shutdown()
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("that only uses write_message", func() {
conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
dRunner.EXPECT().Name().Return("write_message")
err := decoder.Init(conf)
decoder.SetDecoderRunner(dRunner)
c.Assume(err, gs.IsNil)
c.Specify("adds a string field to the message", func() {
data := "string field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(string), gs.Equals, "foo")
})
c.Specify("adds a numeric field to the message", func() {
data := "num field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(float64), gs.Equals, float64(1))
})
c.Specify("adds a boolean field to the message", func() {
data := "bool field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(bool), gs.Equals, true)
})
c.Specify("sets type and payload", func() {
data := "set type and payload"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
})
c.Specify("sets field value with representation", func() {
data := "set field value with representation"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("rep")
c.Expect(len(fields), gs.Equals, 1)
field := fields[0]
values := field.GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "foo")
c.Expect(field.GetRepresentation(), gs.Equals, "representation")
})
c.Specify("sets multiple field string values", func() {
data := "set multiple field string values"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("multi")
c.Expect(len(fields), gs.Equals, 2)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "first")
values = fields[1].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "second")
})
c.Specify("sets field string array value", func() {
data := "set field string array value"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("array")
c.Expect(len(fields), gs.Equals, 1)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 2)
c.Expect(values[0], gs.Equals, "first")
c.Expect(values[1], gs.Equals, "second")
})
})
})
c.Specify("A Multipack SandboxDecoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
supply := make(chan *pipeline.PipelinePack, 3)
pack := pipeline.NewPipelinePack(supply)
pack.Message = getTestMessage()
pack1 := pipeline.NewPipelinePack(supply)
pack2 := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
c.Specify("decodes into multiple packs", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
gomock.InOrder(
dRunner.EXPECT().NewPack().Return(pack1),
dRunner.EXPECT().NewPack().Return(pack2),
)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 3)
c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")
for i := 0; i < 1; i++ {
c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))
}
decoder.Shutdown()
})
})
c.Specify("Linux Cpu Stats decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/linux_loadavg.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes a message", func() {
payload := "0.00 0.01 0.05 3/153 660\n"
pack.Message.SetPayload(payload)
f, err := message.NewField("FilePath", "/proc/loadavg", "")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("1MinAvg")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 0.00)
value, ok = pack.Message.GetFieldValue("5MinAvg")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 0.01)
value, ok = pack.Message.GetFieldValue("15MinAvg")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 0.05)
value, ok = pack.Message.GetFieldValue("NumProcesses")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(3))
value, ok = pack.Message.GetFieldValue("FilePath")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, "/proc/loadavg")
})
c.Specify("decodes an invalid message", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Linux Mem Stats decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/linux_memstats.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes a message", func() {
payload := `MemTotal: 4047616 kB
MemFree: 3135780 kB
HugePages_Free: 0
`
pack.Message.SetPayload(payload)
f, err := message.NewField("FilePath", "/proc/meminfo", "")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("MemTotal")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 4.047616e+06)
value, ok = pack.Message.GetFieldValue("MemFree")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, 3.13578e+06)
value, ok = pack.Message.GetFieldValue("HugePages_Free")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("FilePath")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, "/proc/meminfo")
})
c.Specify("decodes an invalid message", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Linux Disk Stats decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/linux_diskstats.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes a message", func() {
payload := " 13903 11393 969224 49444 10780 10161 1511920 4104 0 5064 53468\n"
pack.Message.SetPayload(payload)
f, err := message.NewField("FilePath", "/sys/block/sda/stat", "")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("TickerInterval", int64(2), "")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
// These are in the same order the payload should be
value, ok = pack.Message.GetFieldValue("ReadsCompleted")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(13903))
value, ok = pack.Message.GetFieldValue("ReadsMerged")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(11393))
value, ok = pack.Message.GetFieldValue("SectorsRead")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(969224))
value, ok = pack.Message.GetFieldValue("TimeReading")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(49444))
value, ok = pack.Message.GetFieldValue("WritesCompleted")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(10780))
value, ok = pack.Message.GetFieldValue("WritesMerged")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(10161))
value, ok = pack.Message.GetFieldValue("SectorsWritten")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(1511920))
value, ok = pack.Message.GetFieldValue("TimeWriting")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(4104))
value, ok = pack.Message.GetFieldValue("NumIOInProgress")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("TimeDoingIO")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(5064))
value, ok = pack.Message.GetFieldValue("WeightedTimeDoingIO")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(53468))
value, ok = pack.Message.GetFieldValue("TickerInterval")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(2))
value, ok = pack.Message.GetFieldValue("FilePath")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, "/sys/block/sda/stat")
})
c.Specify("decodes a message with no leading space", func() {
payload := "19092852 0 510563170 15817012 46452019 0 1546950712 262535124 0 23823976 278362684\n"
pack.Message.SetPayload(payload)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
value, ok := pack.Message.GetFieldValue("ReadsCompleted")
c.Expect(ok, gs.IsTrue)
c.Expect(value, gs.Equals, float64(19092852))
})
c.Specify("decodes an invalid message", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Nginx access log decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Apache access log decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/apache_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("rsyslog decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["type"] = "MyTestFormat"
conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
conf.Config["tz"] = "America/Los_Angeles"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n"
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
// Syslog timestamp doesn't support year, so we have to calculate
// it for the current year or else this test will fail every
// January.
year := time.Now().Year()
tStr := fmt.Sprintf("%d Feb 10 12:58:58 -0800", year)
t, err := time.Parse("2006 Jan 02 15:04:05 -0700", tStr)
c.Assume(err, gs.IsNil)
unixT := t.UnixNano()
c.Expect(pack.Message.GetTimestamp(), gs.Equals, unixT)
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322))
c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.")
c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"])
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("programname")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "widget")
value, ok = pack.Message.GetFieldValue("syslogfacility")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(3))
value, ok = pack.Message.GetFieldValue("timegenerated")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(1392065939000000000))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("mysql decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["truncate_sql"] = int64(5)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decode standard slow query log", func() {
data := `# User@Host: syncrw[syncrw] @ [127.0.0.1]
# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773
use widget;
SET last_insert_id=999,insert_id=1000,timestamp=1399500744;
# administrator command: do something
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;`
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1399500744000000000))
c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query")
decoder.Shutdown()
})
})
c.Specify("mariadb decoder", func() {
decoder := new(SandboxDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["truncate_sql"] = int64(5)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decode standard slow query log", func() {
data := `# User@Host: syncrw[syncrw] @ [127.0.0.1]
# Thread_id: 110804 Schema: weave0 QC_hit: No
# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;`
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1399500744000000000))
c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query")
decoder.Shutdown()
})
})
}
func EncoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
// NewPipelineConfig sets up Globals which is needed for the
// pipeline.Prepend*Dir functions to not die during plugin Init().
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxEncoder", func() {
encoder := new(SandboxEncoder)
encoder.SetPipelineConfig(pConfig)
conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
pack.Message.SetPayload("original")
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321)
pack.Message.SetUuid(uuid.NewRandom())
var (
result []byte
err error
)
c.Specify("emits JSON correctly", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_json.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = json.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
c.Expect(msg.GetPid(), gs.Equals, int32(12345))
c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
c.Expect(msg.GetHostname(), gs.Equals, "hostname")
c.Expect(msg.GetPayload(), gs.Equals, "original")
c.Expect(msg.GetType(), gs.Equals, "my_type")
})
c.Specify("emits text correctly", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_text.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
c.Expect(string(result), gs.Equals, "Prefixed original")
})
c.Specify("emits protobuf correctly", func() {
c.Specify("when inject_message is used", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_protobuf.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = proto.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
c.Expect(msg.GetPid(), gs.Equals, int32(12345))
c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
c.Expect(msg.GetHostname(), gs.Equals, "hostname")
c.Expect(msg.GetPayload(), gs.Equals, "mutated")
c.Expect(msg.GetType(), gs.Equals, "after")
})
c.Specify("when `write_message` is used", func() {
conf.ScriptFilename = "../lua/testsupport/encoder_writemessage.lua"
err = encoder.Init(conf)
c.Expect(err, gs.IsNil)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
msg := new(message.Message)
err = proto.Unmarshal(result, msg)
c.Expect(err, gs.IsNil)
c.Expect(msg.GetPayload(), gs.Equals, "mutated payload")
c.Expect(pack.Message.GetPayload(), gs.Equals, "original")
})
})
})
c.Specify("cbuf librato encoder", func() {
encoder := new(SandboxEncoder)
encoder.SetPipelineConfig(pConfig)
conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321)
pack.Message.SetUuid(uuid.NewRandom())
var (
result []byte
err error
)
conf.ScriptFilename = "../lua/encoders/cbuf_librato.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.Config = make(map[string]interface{})
err = encoder.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("encodes cbuf data", func() {
payload := `{"time":1410823460,"rows":5,"columns":5,"seconds_per_row":5,"column_info":[{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1 2 3 4 5
6 7 8 9 10
11 12 13 14 15
16 17 18 19 20
21 22 23 24 25
`
pack.Message.SetPayload(payload)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `{"gauges":[{"value":1,"measure_time":1410823460,"name":"HTTP_200","source":"hostname"},{"value":2,"measure_time":1410823460,"name":"HTTP_300","source":"hostname"},{"value":3,"measure_time":1410823460,"name":"HTTP_400","source":"hostname"},{"value":4,"measure_time":1410823460,"name":"HTTP_500","source":"hostname"},{"value":5,"measure_time":1410823460,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":6,"measure_time":1410823465,"name":"HTTP_200","source":"hostname"},{"value":7,"measure_time":1410823465,"name":"HTTP_300","source":"hostname"},{"value":8,"measure_time":1410823465,"name":"HTTP_400","source":"hostname"},{"value":9,"measure_time":1410823465,"name":"HTTP_500","source":"hostname"},{"value":10,"measure_time":1410823465,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":11,"measure_time":1410823470,"name":"HTTP_200","source":"hostname"},{"value":12,"measure_time":1410823470,"name":"HTTP_300","source":"hostname"},{"value":13,"measure_time":1410823470,"name":"HTTP_400","source":"hostname"},{"value":14,"measure_time":1410823470,"name":"HTTP_500","source":"hostname"},{"value":15,"measure_time":1410823470,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":16,"measure_time":1410823475,"name":"HTTP_200","source":"hostname"},{"value":17,"measure_time":1410823475,"name":"HTTP_300","source":"hostname"},{"value":18,"measure_time":1410823475,"name":"HTTP_400","source":"hostname"},{"value":19,"measure_time":1410823475,"name":"HTTP_500","source":"hostname"},{"value":20,"measure_time":1410823475,"name":"HTTP_UNKNOWN","source":"hostname"}]}`
c.Expect(string(result), gs.Equals, expected)
c.Specify("and correctly advances", func() {
payload := `{"time":1410823475,"rows":5,"columns":5,"seconds_per_row":5,"column_info":[{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
16 17 18 19 20
21 22 23 24 25
1 2 3 4 5
6 nan 8 nan 10
5 4 3 2 1
`
pack.Message.SetPayload(payload)
result, err = encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `{"gauges":[{"value":21,"measure_time":1410823480,"name":"HTTP_200","source":"hostname"},{"value":22,"measure_time":1410823480,"name":"HTTP_300","source":"hostname"},{"value":23,"measure_time":1410823480,"name":"HTTP_400","source":"hostname"},{"value":24,"measure_time":1410823480,"name":"HTTP_500","source":"hostname"},{"value":25,"measure_time":1410823480,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":1,"measure_time":1410823485,"name":"HTTP_200","source":"hostname"},{"value":2,"measure_time":1410823485,"name":"HTTP_300","source":"hostname"},{"value":3,"measure_time":1410823485,"name":"HTTP_400","source":"hostname"},{"value":4,"measure_time":1410823485,"name":"HTTP_500","source":"hostname"},{"value":5,"measure_time":1410823485,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":6,"measure_time":1410823490,"name":"HTTP_200","source":"hostname"},{"value":8,"measure_time":1410823490,"name":"HTTP_400","source":"hostname"},{"value":10,"measure_time":1410823490,"name":"HTTP_UNKNOWN","source":"hostname"}]}`
c.Expect(string(result), gs.Equals, expected)
})
})
})
c.Specify("schema influx encoder", func() {
encoder := new(SandboxEncoder)
encoder.SetPipelineConfig(pConfig)
conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
pack.Message.SetType("my_type")
pack.Message.SetPid(12345)
pack.Message.SetSeverity(4)
pack.Message.SetHostname("hostname")
pack.Message.SetTimestamp(54321 * 1e9)
pack.Message.SetLogger("Logger")
pack.Message.SetPayload("Payload value lorem ipsum")
f, err := message.NewField("intField", 123, "")
c.Assume(err, gs.IsNil)
err = f.AddValue(456)
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("strField", "0_first", "")
c.Assume(err, gs.IsNil)
err = f.AddValue("0_second")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("strField", "1_first", "")
c.Assume(err, gs.IsNil)
err = f.AddValue("1_second")
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
f, err = message.NewField("byteField", []byte("first"), "")
c.Assume(err, gs.IsNil)
err = f.AddValue([]byte("second"))
c.Assume(err, gs.IsNil)
pack.Message.AddField(f)
conf.ScriptFilename = "../lua/encoders/schema_influx.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.Config = make(map[string]interface{})
c.Specify("encodes a basic message", func() {
err = encoder.Init(conf)
c.Assume(err, gs.IsNil)
result, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `[{"points":[[54321000,"my_type","Payload value lorem ipsum","hostname",12345,"Logger",4,"",[123,456],["0_first","0_second"],["1_first","1_second"]]],"name":"series","columns":["time","Type","Payload","Hostname","Pid","Logger","Severity","EnvVersion","intField","strField","strField2"]}]`
c.Expect(string(result), gs.Equals, expected)
})
c.Specify("interpolates series name correctly", func() {
conf.Config["series"] = "series.%{Pid}.%{Type}.%{strField}.%{intField}"
err = encoder.Init(conf)
c.Assume(err, gs.IsNil)
result, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `[{"points":[[54321000,"my_type","Payload value lorem ipsum","hostname",12345,"Logger",4,"",[123,456],["0_first","0_second"],["1_first","1_second"]]],"name":"series.12345.my_type.0_first.123","columns":["time","Type","Payload","Hostname","Pid","Logger","Severity","EnvVersion","intField","strField","strField2"]}]`
c.Expect(string(result), gs.Equals, expected)
})
c.Specify("skips specified correctly", func() {
conf.Config["skip_fields"] = "Payload strField Type"
err = encoder.Init(conf)
c.Assume(err, gs.IsNil)
result, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
expected := `[{"points":[[54321000,"hostname",12345,"Logger",4,"",[123,456]]],"name":"series","columns":["time","Hostname","Pid","Logger","Severity","EnvVersion","intField"]}]`
c.Expect(string(result), gs.Equals, expected)
})
})
}
func main() {
configPath := flag.String("config", filepath.FromSlash("/etc/hekad.toml"),
"Config file or directory. If directory is specified then all files "+
"in the directory will be loaded.")
version := flag.Bool("version", false, "Output version and exit")
flag.Parse()
config := &HekadConfig{}
var err error
var cpuProfName string
var memProfName string
if *version {
fmt.Println(VERSION)
os.Exit(0)
}
config, err = LoadHekadConfig(*configPath)
if err != nil {
pipeline.LogError.Fatal("Error reading config: ", err)
}
if config.SampleDenominator <= 0 {
pipeline.LogError.Fatalln("'sample_denominator' value must be greater than 0.")
}
globals, cpuProfName, memProfName := setGlobalConfigs(config)
if err = os.MkdirAll(globals.BaseDir, 0755); err != nil {
pipeline.LogError.Fatalf("Error creating 'base_dir' %s: %s", config.BaseDir, err)
}
if config.MaxMessageSize > 1024 {
message.SetMaxMessageSize(config.MaxMessageSize)
} else if config.MaxMessageSize > 0 {
pipeline.LogError.Fatalln("Error: 'max_message_size' setting must be greater than 1024.")
}
if config.PidFile != "" {
contents, err := ioutil.ReadFile(config.PidFile)
if err == nil {
pid, err := strconv.Atoi(strings.TrimSpace(string(contents)))
if err != nil {
pipeline.LogError.Fatalf("Error reading proccess id from pidfile '%s': %s",
config.PidFile, err)
}
process, err := os.FindProcess(pid)
// on Windows, err != nil if the process cannot be found
if runtime.GOOS == "windows" {
if err == nil {
pipeline.LogError.Fatalf("Process %d is already running.", pid)
}
} else if process != nil {
// err is always nil on POSIX, so we have to send the process
// a signal to check whether it exists
if err = process.Signal(syscall.Signal(0)); err == nil {
pipeline.LogError.Fatalf("Process %d is already running.", pid)
}
}
}
if err = ioutil.WriteFile(config.PidFile, []byte(strconv.Itoa(os.Getpid())),
0644); err != nil {
pipeline.LogError.Fatalf("Unable to write pidfile '%s': %s", config.PidFile, err)
}
pipeline.LogInfo.Printf("Wrote pid to pidfile '%s'", config.PidFile)
defer func() {
if err = os.Remove(config.PidFile); err != nil {
pipeline.LogError.Printf("Unable to remove pidfile '%s': %s", config.PidFile, err)
}
}()
}
if cpuProfName != "" {
profFile, err := os.Create(cpuProfName)
if err != nil {
pipeline.LogError.Fatalln(err)
}
pprof.StartCPUProfile(profFile)
defer func() {
pprof.StopCPUProfile()
profFile.Close()
}()
}
if memProfName != "" {
defer func() {
profFile, err := os.Create(memProfName)
if err != nil {
pipeline.LogError.Fatalln(err)
}
pprof.WriteHeapProfile(profFile)
profFile.Close()
}()
}
// Set up and load the pipeline configuration and start the daemon.
pipeconf := pipeline.NewPipelineConfig(globals)
if err = loadFullConfig(pipeconf, configPath); err != nil {
pipeline.LogError.Fatal("Error reading config: ", err)
}
pipeline.Run(pipeconf)
}
func DecoderSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
// NewPipelineConfig sets up Globals which is needed for the
// pipeline.Prepend*Dir functions to not die during plugin Init().
_ = pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
c.Specify("that uses lpeg and inject_message", func() {
dRunner.EXPECT().Name().Return("serialize")
conf.ScriptFilename = "../lua/testsupport/decoder.lua"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
c.Specify("decodes simple messages", func() {
data := "1376389920 debug id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1376389920000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("id")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "2321")
value, ok = pack.Message.GetFieldValue("url")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "example.com")
value, ok = pack.Message.GetFieldValue("item")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "1")
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "1376389920 bogus id=2321 url=example.com item=1"
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
c.Specify("Preserves data", func() {
conf.ScriptFilename = "../lua/testsupport/serialize.lua"
conf.PreserveData = true
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
decoder.Shutdown()
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("that only uses write_message", func() {
conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
dRunner.EXPECT().Name().Return("write_message")
err := decoder.Init(conf)
decoder.SetDecoderRunner(dRunner)
c.Assume(err, gs.IsNil)
c.Specify("adds a string field to the message", func() {
data := "string field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(string), gs.Equals, "foo")
})
c.Specify("adds a numeric field to the message", func() {
data := "num field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(float64), gs.Equals, float64(1))
})
c.Specify("adds a boolean field to the message", func() {
data := "bool field scribble"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
value, ok := pack.Message.GetFieldValue("scribble")
c.Expect(ok, gs.IsTrue)
c.Expect(value.(bool), gs.Equals, true)
})
c.Specify("sets type and payload", func() {
data := "set type and payload"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
})
c.Specify("sets field value with representation", func() {
data := "set field value with representation"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("rep")
c.Expect(len(fields), gs.Equals, 1)
field := fields[0]
values := field.GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "foo")
c.Expect(field.GetRepresentation(), gs.Equals, "representation")
})
c.Specify("sets multiple field string values", func() {
data := "set multiple field string values"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("multi")
c.Expect(len(fields), gs.Equals, 2)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "first")
values = fields[1].GetValueString()
c.Expect(len(values), gs.Equals, 1)
c.Expect(values[0], gs.Equals, "second")
})
c.Specify("sets field string array value", func() {
data := "set field string array value"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 1)
c.Expect(packs[0], gs.Equals, pack)
fields := pack.Message.FindAllFields("array")
c.Expect(len(fields), gs.Equals, 1)
values := fields[0].GetValueString()
c.Expect(len(values), gs.Equals, 2)
c.Expect(values[0], gs.Equals, "first")
c.Expect(values[1], gs.Equals, "second")
})
})
})
c.Specify("A Multipack SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
supply := make(chan *pipeline.PipelinePack, 3)
pack := pipeline.NewPipelinePack(supply)
pack.Message = getTestMessage()
pack1 := pipeline.NewPipelinePack(supply)
pack2 := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
c.Specify("decodes into multiple packs", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
gomock.InOrder(
dRunner.EXPECT().NewPack().Return(pack1),
dRunner.EXPECT().NewPack().Return(pack2),
)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 3)
c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")
for i := 0; i < 1; i++ {
c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))
}
decoder.Shutdown()
})
})
c.Specify("Nginx access log decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("Apache access log decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/apache_access.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
conf.Config["user_agent_transform"] = true
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392050801000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("remote_addr")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "127.0.0.1")
value, ok = pack.Message.GetFieldValue("user_agent_browser")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Firefox")
value, ok = pack.Message.GetFieldValue("user_agent_version")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(26))
value, ok = pack.Message.GetFieldValue("user_agent_os")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "Linux")
_, ok = pack.Message.GetFieldValue("http_user_agent")
c.Expect(ok, gs.Equals, false)
value, ok = pack.Message.GetFieldValue("body_bytes_sent")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(0))
value, ok = pack.Message.GetFieldValue("status")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(304))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("rsyslog decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["type"] = "MyTestFormat"
conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
conf.Config["tz"] = "America/Los_Angeles"
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decodes simple messages", func() {
data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n"
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1392065938000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322))
c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.")
c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"])
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("programname")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "widget")
value, ok = pack.Message.GetFieldValue("syslogfacility")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(3))
value, ok = pack.Message.GetFieldValue("timegenerated")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, float64(1392065939000000000))
decoder.Shutdown()
})
c.Specify("decodes an invalid messages", func() {
data := "bogus message"
pack.Message.SetPayload(data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
decoder.Shutdown()
})
})
c.Specify("mysql decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["truncate_sql"] = int64(5)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decode standard slow query log", func() {
data := `# User@Host: syncrw[syncrw] @ [127.0.0.1]
# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773
use widget;
SET last_insert_id=999,insert_id=1000,timestamp=1399500744;
# administrator command: do something
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;
`
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1399500744000000000))
c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query")
decoder.Shutdown()
})
})
c.Specify("mariadb decoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua"
conf.ModuleDirectory = "../../../../../../modules"
conf.MemoryLimit = 8e6
conf.Config = make(map[string]interface{})
conf.Config["truncate_sql"] = int64(5)
supply := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(supply)
dRunner := pm.NewMockDecoderRunner(ctrl)
dRunner.EXPECT().Name().Return("SandboxDecoder")
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("decode standard slow query log", func() {
data := `# User@Host: syncrw[syncrw] @ [127.0.0.1]
# Thread_id: 110804 Schema: weave0 QC_hit: No
# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;
`
pack.Message.SetPayload(data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1399500744000000000))
c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query")
decoder.Shutdown()
})
})
}
func FilterSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
fth := NewFilterTestHelper(ctrl)
inChan := make(chan *pipeline.PipelinePack, 1)
pConfig := pipeline.NewPipelineConfig(nil)
c.Specify("A SandboxFilter", func() {
sbFilter := new(SandboxFilter)
config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
config.ScriptType = "lua"
config.MemoryLimit = 32000
config.InstructionLimit = 1000
config.OutputLimit = 1024
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("Over inject messages from ProcessMessage", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(3)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(2)
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("exceeded InjectMessage count"))
config.ScriptFilename = "../lua/testsupport/processinject.lua"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
close(inChan)
sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
})
c.Specify("Over inject messages from TimerEvent", func() {
var timer <-chan time.Time
timer = time.Tick(time.Duration(1) * time.Millisecond)
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(12)
fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(11)
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("exceeded InjectMessage count"))
config.ScriptFilename = "../lua/testsupport/timerinject.lua"
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
go func() {
time.Sleep(time.Duration(250) * time.Millisecond)
close(inChan)
}()
sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
})
c.Specify("Preserves data", func() {
var timer <-chan time.Time
fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
config.ScriptFilename = "../lua/testsupport/serialize.lua"
config.PreserveData = true
sbFilter.SetName("serialize")
err := sbFilter.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
_, err = os.Stat("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
err = os.Remove("sandbox_preservation/serialize.data")
c.Expect(err, gs.IsNil)
})
})
c.Specify("A SandboxManagerFilter", func() {
sbmFilter := new(SandboxManagerFilter)
config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
config.MaxFilters = 1
origBaseDir := pipeline.Globals().BaseDir
pipeline.Globals().BaseDir = os.TempDir()
sbxMgrsDir := filepath.Join(pipeline.Globals().BaseDir, "sbxmgrs")
defer func() {
pipeline.Globals().BaseDir = origBaseDir
tmpErr := os.RemoveAll(sbxMgrsDir)
c.Expect(tmpErr, gs.IsNil)
}()
msg := getTestMessage()
pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("Control message in the past", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: 5 seconds skew"))
inChan <- pack
close(inChan)
sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
})
c.Specify("Control message in the future", func() {
sbmFilter.Init(config)
pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: -5 seconds skew"))
inChan <- pack
close(inChan)
sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
})
c.Specify("Generates the right default working directory", func() {
sbmFilter.Init(config)
fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
name := "SandboxManagerFilter"
fth.MockFilterRunner.EXPECT().Name().Return(name)
close(inChan)
sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
_, err := os.Stat(sbxMgrsDir)
c.Expect(err, gs.IsNil)
})
c.Specify("Sanity check the default sandbox configuration limits", func() {
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
})
c.Specify("Sanity check the user specified sandbox configuration limits", func() {
config.MemoryLimit = 123456
config.InstructionLimit = 4321
config.OutputLimit = 8765
sbmFilter.Init(config)
c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
})
})
}
func NsqInputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
var wg sync.WaitGroup
errChan := make(chan error, 1)
retPackChan := make(chan *pipeline.PipelinePack, 1)
defer close(errChan)
defer close(retPackChan)
c.Specify("A nsq input", func() {
input := new(NsqInput)
ith := new(plugins_ts.InputTestHelper)
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
config := input.ConfigStruct().(*NsqInputConfig)
config.Topic = "test_topic"
config.Channel = "test_channel"
startInput := func() {
wg.Add(1)
go func() {
errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
wg.Done()
}()
}
var mockConsumer *MockConsumer
input.newConsumer = func(topic, channel string, config *nsq.Config) (Consumer, error) {
mockConsumer, _ = NewMockConsumer(topic, channel, config)
return mockConsumer, nil
}
ith.Pack = pipeline.NewPipelinePack(pConfig.InputRecycleChan())
ith.PackSupply = make(chan *pipeline.PipelinePack, 1)
ith.PackSupply <- ith.Pack
inputName := "NsqInput"
ith.MockInputRunner.EXPECT().Name().Return(inputName).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
c.Specify("that is started", func() {
input.SetPipelineConfig(pConfig)
c.Specify("gets messages its subscribed to", func() {
c.Specify("injects messages into the pipeline when not configured with a decoder", func() {
ith.MockInputRunner.EXPECT().Inject(ith.Pack).Do(func(p *pipeline.PipelinePack) {
retPackChan <- p
}).AnyTimes()
err := input.Init(config)
c.Expect(err, gs.IsNil)
c.Expect(input.DecoderName, gs.Equals, "")
})
c.Specify("injects messages into the decoder when configured", func() {
decoderName := "ScribbleDecoder"
config.DecoderName = decoderName
decoder := new(plugins.ScribbleDecoder)
decoder.Init(&plugins.ScribbleDecoderConfig{})
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
mockDecoderRunner.EXPECT().InChan().Return(retPackChan)
ith.MockHelper.EXPECT().DecoderRunner(decoderName,
fmt.Sprintf("%s-%s", inputName, decoderName),
).Return(mockDecoderRunner, true)
err := input.Init(config)
c.Expect(err, gs.IsNil)
c.Expect(input.DecoderName, gs.Equals, decoderName)
})
startInput()
// Should get two finished conn calls since we connect
// to lookupds and nsqds
c.Expect(<-mockConsumer.finishedConn, gs.IsTrue)
c.Expect(<-mockConsumer.finishedConn, gs.IsTrue)
id := nsq.MessageID{}
body := []byte("test")
msg := nsq.NewMessage(id, body)
for _, handler := range mockConsumer.handlers {
handler.HandleMessage(msg)
}
p := <-retPackChan
c.Expect(p.Message.GetPayload(), gs.Equals, "test")
})
input.Stop()
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
})
})
}
