func HttpListenInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) httpListenInput := HttpListenInput{} ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) startInput := func() { go func() { httpListenInput.Run(ith.MockInputRunner, ith.MockHelper) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) config := httpListenInput.ConfigStruct().(*HttpListenInputConfig) config.Address = "127.0.0.1:8325" config.Decoder = "PayloadJsonDecoder" ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan).AnyTimes() ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes() ith.MockInputRunner.EXPECT().Name().Return("HttpListenInput").AnyTimes() ith.MockHelper.EXPECT().DecoderRunner("PayloadJsonDecoder", "HttpListenInput-PayloadJsonDecoder").Return(mockDecoderRunner, true) err := httpListenInput.Init(config) c.Assume(err, gs.IsNil) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()) startInput() c.Specify("A HttpListenInput", func() { c.Specify("Adds query parameters to the message pack as fields", func() { ith.PackSupply <- ith.Pack resp, err := http.Get("http://127.0.0.1:8325/?test=Hello%20World") c.Assume(err, gs.IsNil) resp.Body.Close() c.Assume(resp.StatusCode, gs.Equals, 200) pack := <-dRunnerInChan fieldValue, ok := pack.Message.GetFieldValue("test") c.Assume(ok, gs.IsTrue) c.Expect(fieldValue, gs.Equals, "Hello World") }) httpListenInput.Stop() }) }
// decodeMessageAndVerifyOutput takes a decoder conf, message payload, and a fn -> the fn is a number of // assertions to verify that the message after decoding is as expected. func decodeMessageAndVerifyOutput(c gs.Context, conf *JsonDecoderConfig, payload string, fn packVerifier) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() // 1. Initialize test decoder decoder := new(JsonDecoder) err := decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) // 2. Set payload to be tested, and decode it supply := make(chan *PipelinePack, 1) pack := NewPipelinePack(supply) pack.Message.SetPayload(payload) _, err = decoder.Decode(pack) // 3. Assert outcome of decoding fn(c, pack) pack.Zero() }
func DecoderSpec(c gs.Context) { t := new(ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() // NewPipelineConfig sets up Globals which is needed for the // pipeline.Prepend*Dir functions to not die during plugin Init(). _ = pipeline.NewPipelineConfig(nil) c.Specify("A SandboxDecoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) c.Specify("that uses lpeg and inject_message", func() { dRunner.EXPECT().Name().Return("serialize") conf.ScriptFilename = "../lua/testsupport/decoder.lua" err := decoder.Init(conf) c.Assume(err, gs.IsNil) c.Specify("decodes simple messages", func() { data := "1376389920 debug id=2321 url=example.com item=1" decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1376389920000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("id") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "2321") value, ok = pack.Message.GetFieldValue("url") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "example.com") value, ok = pack.Message.GetFieldValue("item") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "1") decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "1376389920 bogus id=2321 url=example.com item=1" decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) c.Specify("Preserves data", func() { conf.ScriptFilename = "../lua/testsupport/serialize.lua" conf.PreserveData = true err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) decoder.Shutdown() _, err = os.Stat("sandbox_preservation/serialize.data") c.Expect(err, gs.IsNil) err = os.Remove("sandbox_preservation/serialize.data") c.Expect(err, gs.IsNil) }) }) c.Specify("that only uses write_message", func() { conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua" dRunner.EXPECT().Name().Return("write_message") err := decoder.Init(conf) decoder.SetDecoderRunner(dRunner) c.Assume(err, gs.IsNil) c.Specify("adds a string field to the message", func() { data := "string field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(string), gs.Equals, "foo") }) c.Specify("adds a numeric field to the message", func() { data := "num field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(float64), gs.Equals, float64(1)) }) c.Specify("adds a boolean field to the message", func() { data := "bool field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(bool), gs.Equals, true) }) c.Specify("sets type and payload", func() { data := "set type and payload" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) c.Expect(pack.Message.GetType(), gs.Equals, "my_type") c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload") }) c.Specify("sets field value with representation", func() { data := "set field value with representation" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("rep") c.Expect(len(fields), gs.Equals, 1) field := fields[0] values := field.GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "foo") c.Expect(field.GetRepresentation(), gs.Equals, "representation") }) c.Specify("sets multiple field string values", func() { data := "set multiple field string values" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("multi") c.Expect(len(fields), gs.Equals, 2) values := fields[0].GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "first") values = fields[1].GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "second") }) c.Specify("sets field string array value", func() { data := "set field string array value" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("array") c.Expect(len(fields), gs.Equals, 1) values := fields[0].GetValueString() c.Expect(len(values), gs.Equals, 2) c.Expect(values[0], gs.Equals, "first") c.Expect(values[1], gs.Equals, "second") }) }) }) c.Specify("A Multipack SandboxDecoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua" supply := make(chan *pipeline.PipelinePack, 3) pack := pipeline.NewPipelinePack(supply) pack.Message = getTestMessage() pack1 := pipeline.NewPipelinePack(supply) pack2 := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") c.Specify("decodes into multiple packs", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) gomock.InOrder( dRunner.EXPECT().NewPack().Return(pack1), dRunner.EXPECT().NewPack().Return(pack2), ) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 3) c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one") c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two") c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three") for i := 0; i < 1; i++ { c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST") c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name") c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec") c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6)) } decoder.Shutdown() }) }) c.Specify("Nginx access log decoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/nginx_access.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\"" conf.Config["user_agent_transform"] = true supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes simple messages", func() { data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\"" pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1392050801000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("remote_addr") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "127.0.0.1") value, ok = pack.Message.GetFieldValue("user_agent_browser") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Firefox") value, ok = pack.Message.GetFieldValue("user_agent_version") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(26)) value, ok = pack.Message.GetFieldValue("user_agent_os") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Linux") _, ok = pack.Message.GetFieldValue("http_user_agent") c.Expect(ok, gs.Equals, false) value, ok = pack.Message.GetFieldValue("body_bytes_sent") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(0)) value, ok = pack.Message.GetFieldValue("status") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(304)) decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("Apache access log decoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/apache_access.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" conf.Config["user_agent_transform"] = true supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes simple messages", func() { data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\"" pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1392050801000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("remote_addr") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "127.0.0.1") value, ok = pack.Message.GetFieldValue("user_agent_browser") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Firefox") value, ok = pack.Message.GetFieldValue("user_agent_version") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(26)) value, ok = pack.Message.GetFieldValue("user_agent_os") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Linux") _, ok = pack.Message.GetFieldValue("http_user_agent") c.Expect(ok, gs.Equals, false) value, ok = pack.Message.GetFieldValue("body_bytes_sent") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(0)) value, ok = pack.Message.GetFieldValue("status") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(304)) decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("rsyslog decoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/rsyslog.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["type"] = "MyTestFormat" conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" conf.Config["tz"] = "America/Los_Angeles" supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes simple messages", func() { data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n" pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1392065938000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4)) c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost") c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322)) c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.") c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"]) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("programname") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "widget") value, ok = pack.Message.GetFieldValue("syslogfacility") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(3)) value, ok = pack.Message.GetFieldValue("timegenerated") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(1392065939000000000)) decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("mysql decoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["truncate_sql"] = int64(5) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decode standard slow query log", func() { data := `# User@Host: syncrw[syncrw] @ [127.0.0.1] # Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 use widget; SET last_insert_id=999,insert_id=1000,timestamp=1399500744; # administrator command: do something /* [queryName=FIND_ITEMS] */ SELECT * FROM widget WHERE id = 10; ` pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1399500744000000000)) c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...") c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query") decoder.Shutdown() }) }) c.Specify("mariadb decoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["truncate_sql"] = int64(5) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decode standard slow query log", func() { data := `# User@Host: syncrw[syncrw] @ [127.0.0.1] # Thread_id: 110804 Schema: weave0 QC_hit: No # Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198 SET timestamp=1399500744; /* [queryName=FIND_ITEMS] */ SELECT * FROM widget WHERE id = 10; ` pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1399500744000000000)) c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...") c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query") decoder.Shutdown() }) }) }
func TcpInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) key := "testkey" signers := map[string]Signer{"test_1": {key}} signer := "test" c.Specify("A TcpInput protobuf parser", func() { ith.MockInputRunner.EXPECT().Name().Return("TcpInput") tcpInput := TcpInput{} err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr, Signers: signers, Decoder: "ProtobufDecoder", ParserType: "message.proto"}) c.Assume(err, gs.IsNil) realListener := tcpInput.listener c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr) realListener.Close() mockConnection := pipeline_ts.NewMockConn(ctrl) mockListener := pipeline_ts.NewMockListener(ctrl) tcpInput.listener = mockListener addr := new(address) addr.str = "123" mockConnection.EXPECT().RemoteAddr().Return(addr) mbytes, _ := proto.Marshal(ith.Msg) header := &message.Header{} header.SetMessageLength(uint32(len(mbytes))) err = errors.New("connection closed") // used in the read return(s) readCall := mockConnection.EXPECT().Read(gomock.Any()) readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall) readEnd.Return(0, err) mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes() mockConnection.EXPECT().Close() neterr := pipeline_ts.NewMockError(ctrl) neterr.EXPECT().Temporary().Return(false) acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil) acceptCall.Do(func() { acceptCall = mockListener.EXPECT().Accept() acceptCall.Return(nil, neterr) }) mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner) mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) enccall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "TcpInput-123-ProtobufDecoder").AnyTimes() enccall.Return(ith.Decoder, true) ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder) cleanup := func() { mockListener.EXPECT().Close() tcpInput.Stop() tcpInput.wg.Wait() } c.Specify("reads a message from its connection", func() { hbytes, _ := proto.Marshal(header) buflen := 3 + len(hbytes) + len(mbytes) readCall.Return(buflen, nil) readCall.Do(getPayloadBytes(hbytes, mbytes)) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer cleanup() ith.PackSupply <- ith.Pack packRef := <-ith.DecodeChan c.Expect(ith.Pack, gs.Equals, packRef) c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes)) }) c.Specify("reads a MD5 signed message from its connection", func() { header.SetHmacHashFunction(message.Header_MD5) header.SetHmacSigner(signer) header.SetHmacKeyVersion(uint32(1)) hm := hmac.New(md5.New, []byte(key)) hm.Write(mbytes) header.SetHmac(hm.Sum(nil)) hbytes, _ := proto.Marshal(header) buflen := 3 + len(hbytes) + len(mbytes) readCall.Return(buflen, nil) readCall.Do(getPayloadBytes(hbytes, mbytes)) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer cleanup() ith.PackSupply <- ith.Pack timeout := make(chan bool, 1) go func() { time.Sleep(100 * time.Millisecond) timeout <- true }() select { case packRef := <-ith.DecodeChan: c.Expect(ith.Pack, gs.Equals, packRef) c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes)) c.Expect(ith.Pack.Signer, gs.Equals, "test") case t := <-timeout: c.Expect(t, gs.IsNil) } }) c.Specify("reads a SHA1 signed message from its connection", func() { header.SetHmacHashFunction(message.Header_SHA1) header.SetHmacSigner(signer) header.SetHmacKeyVersion(uint32(1)) hm := hmac.New(sha1.New, []byte(key)) hm.Write(mbytes) header.SetHmac(hm.Sum(nil)) hbytes, _ := proto.Marshal(header) buflen := 3 + len(hbytes) + len(mbytes) readCall.Return(buflen, nil) readCall.Do(getPayloadBytes(hbytes, mbytes)) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer cleanup() ith.PackSupply <- ith.Pack timeout := make(chan bool, 1) go func() { time.Sleep(100 * time.Millisecond) timeout <- true }() select { case packRef := <-ith.DecodeChan: c.Expect(ith.Pack, gs.Equals, packRef) c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes)) c.Expect(ith.Pack.Signer, gs.Equals, "test") case t := <-timeout: c.Expect(t, gs.IsNil) } }) c.Specify("reads a signed message with an expired key from its connection", func() { header.SetHmacHashFunction(message.Header_MD5) header.SetHmacSigner(signer) header.SetHmacKeyVersion(uint32(11)) // non-existent key version hm := hmac.New(md5.New, []byte(key)) hm.Write(mbytes) header.SetHmac(hm.Sum(nil)) hbytes, _ := proto.Marshal(header) buflen := 3 + len(hbytes) + len(mbytes) readCall.Return(buflen, nil) readCall.Do(getPayloadBytes(hbytes, mbytes)) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer cleanup() ith.PackSupply <- ith.Pack timeout := make(chan bool, 1) go func() { time.Sleep(100 * time.Millisecond) timeout <- true }() select { case packRef := <-mockDecoderRunner.InChan(): c.Expect(packRef, gs.IsNil) case t := <-timeout: c.Expect(t, gs.IsTrue) } }) c.Specify("reads a signed message with an incorrect hmac from its connection", func() { header.SetHmacHashFunction(message.Header_MD5) header.SetHmacSigner(signer) header.SetHmacKeyVersion(uint32(1)) hm := hmac.New(md5.New, []byte(key)) hm.Write([]byte("some bytes")) header.SetHmac(hm.Sum(nil)) hbytes, _ := proto.Marshal(header) buflen := 3 + len(hbytes) + len(mbytes) readCall.Return(buflen, nil) readCall.Do(getPayloadBytes(hbytes, mbytes)) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer cleanup() ith.PackSupply <- ith.Pack timeout := make(chan bool, 1) go func() { time.Sleep(100 * time.Millisecond) timeout <- true }() select { case packRef := <-mockDecoderRunner.InChan(): c.Expect(packRef, gs.IsNil) case t := <-timeout: c.Expect(t, gs.IsTrue) } }) }) c.Specify("A TcpInput regexp parser", func() { ith.MockInputRunner.EXPECT().Name().Return("TcpInput") config := &TcpInputConfig{ Net: "tcp", Address: ith.AddrStr, Decoder: "RegexpDecoder", ParserType: "regexp", } tcpInput := TcpInput{} err := tcpInput.Init(config) c.Assume(err, gs.IsNil) realListener := tcpInput.listener c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr) realListener.Close() mockConnection := pipeline_ts.NewMockConn(ctrl) mockListener := pipeline_ts.NewMockListener(ctrl) tcpInput.listener = mockListener addr := new(address) addr.str = "123" mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2) mbytes := []byte("this is a test message\n") err = errors.New("connection closed") // used in the read return(s) readCall := mockConnection.EXPECT().Read(gomock.Any()) readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall) readEnd.Return(0, err) mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes() mockConnection.EXPECT().Close() neterr := pipeline_ts.NewMockError(ctrl) neterr.EXPECT().Temporary().Return(false) acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil) acceptCall.Do(func() { acceptCall = mockListener.EXPECT().Accept() acceptCall.Return(nil, neterr) }) mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner) mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Name().Return("logger") enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "TcpInput-123-RegexpDecoder").AnyTimes() enccall.Return(ith.Decoder, true) ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder) c.Specify("reads a message from its connection", func() { readCall.Return(len(mbytes), nil) readCall.Do(getPayloadText(mbytes)) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer func() { mockListener.EXPECT().Close() tcpInput.Stop() tcpInput.wg.Wait() }() ith.PackSupply <- ith.Pack packRef := <-ith.DecodeChan c.Expect(ith.Pack, gs.Equals, packRef) c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes[:len(mbytes)-1])) c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger") c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123") }) }) c.Specify("A TcpInput token parser", func() { ith.MockInputRunner.EXPECT().Name().Return("TcpInput") tcpInput := TcpInput{} err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr, Decoder: "TokenDecoder", ParserType: "token", Delimiter: "\n"}) c.Assume(err, gs.IsNil) realListener := tcpInput.listener c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr) realListener.Close() mockConnection := pipeline_ts.NewMockConn(ctrl) mockListener := pipeline_ts.NewMockListener(ctrl) tcpInput.listener = mockListener addr := new(address) addr.str = "123" mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2) mbytes := []byte("this is a test message\n") err = errors.New("connection closed") // used in the read return(s) readCall := mockConnection.EXPECT().Read(gomock.Any()) readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall) readEnd.Return(0, err) mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes() mockConnection.EXPECT().Close() neterr := pipeline_ts.NewMockError(ctrl) neterr.EXPECT().Temporary().Return(false) acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil) acceptCall.Do(func() { acceptCall = mockListener.EXPECT().Accept() acceptCall.Return(nil, neterr) }) mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner) mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Name().Return("logger") enccall := ith.MockHelper.EXPECT().DecoderRunner("TokenDecoder", "TcpInput-123-TokenDecoder").AnyTimes() enccall.Return(ith.Decoder, true) ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder) c.Specify("reads a message from its connection", func() { readCall.Return(len(mbytes), nil) readCall.Do(getPayloadText(mbytes)) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer func() { mockListener.EXPECT().Close() tcpInput.Stop() tcpInput.wg.Wait() }() ith.PackSupply <- ith.Pack packRef := <-ith.DecodeChan c.Expect(ith.Pack, gs.Equals, packRef) c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes)) c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger") c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123") }) }) c.Specify("A TcpInput using TLS", func() { tcpInput := TcpInput{} config := &TcpInputConfig{ Net: "tcp", Address: ith.AddrStr, ParserType: "token", UseTls: true, } c.Specify("fails to init w/ missing key or cert file", func() { config.Tls = TlsConfig{} err := tcpInput.Init(config) c.Expect(err, gs.Not(gs.IsNil)) }) c.Specify("accepts TLS client connections", func() { ith.MockInputRunner.EXPECT().Name().Return("TcpInput") config.Tls = TlsConfig{ CertFile: "./testsupport/cert.pem", KeyFile: "./testsupport/key.pem", } err := tcpInput.Init(config) c.Expect(err, gs.IsNil) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer func() { tcpInput.Stop() tcpInput.wg.Wait() }() clientConfig := new(tls.Config) clientConfig.InsecureSkipVerify = true conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig) c.Expect(err, gs.IsNil) defer conn.Close() conn.SetWriteDeadline(time.Now().Add(time.Duration(10000))) n, err := conn.Write([]byte("This is a test.")) c.Expect(err, gs.IsNil) c.Expect(n, gs.Equals, len("This is a test.")) }) c.Specify("doesn't accept connections below specified min TLS version", func() { ith.MockInputRunner.EXPECT().Name().Return("TcpInput") config.Tls = TlsConfig{ CertFile: "./testsupport/cert.pem", KeyFile: "./testsupport/key.pem", MinVersion: "TLS12", } err := tcpInput.Init(config) c.Expect(err, gs.IsNil) go tcpInput.Run(ith.MockInputRunner, ith.MockHelper) defer func() { tcpInput.Stop() tcpInput.wg.Wait() time.Sleep(time.Duration(1000)) }() clientConfig := &tls.Config{ InsecureSkipVerify: true, MaxVersion: tls.VersionTLS11, } conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig) c.Expect(conn, gs.IsNil) c.Expect(err, gs.Not(gs.IsNil)) }) }) }
func MultiDecoderSpec(c gospec.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() NewPipelineConfig(nil) // initializes Globals() c.Specify("A MultiDecoder", func() { decoder := new(MultiDecoder) decoder.SetName("MyMultiDecoder") conf := decoder.ConfigStruct().(*MultiDecoderConfig) supply := make(chan *PipelinePack, 1) pack := NewPipelinePack(supply) conf.Subs = make(map[string]interface{}, 0) conf.Subs["StartsWithM"] = make(map[string]interface{}, 0) withM := conf.Subs["StartsWithM"].(map[string]interface{}) withM["type"] = "PayloadRegexDecoder" withM["match_regex"] = "^(?P<TheData>m.*)" withMFields := make(map[string]interface{}, 0) withMFields["StartsWithM"] = "%TheData%" withM["message_fields"] = withMFields conf.Order = []string{"StartsWithM"} errMsg := "All subdecoders failed." dRunner := pipelinemock.NewMockDecoderRunner(ctrl) // An error will be spit out b/c there's no real *dRunner in there; // doesn't impact the tests. dRunner.EXPECT().LogError(gomock.Any()) c.Specify("decodes simple messages", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "matching text" pack.Message.SetPayload(regex_data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetType(), gs.Equals, "heka.MyMultiDecoder") value, ok := pack.Message.GetFieldValue("StartsWithM") c.Assume(ok, gs.IsTrue) c.Expect(value, gs.Equals, regex_data) }) c.Specify("returns an error if all decoders fail", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "non-matching text" pack.Message.SetPayload(regex_data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) }) c.Specify("logs subdecoder failures when configured to do so", func() { conf.LogSubErrors = true err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "non-matching text" pack.Message.SetPayload(regex_data) // Expect that we log an error for undecoded message. dRunner.EXPECT().LogError(fmt.Errorf( "Subdecoder 'StartsWithM' decode error: No match: %s", regex_data)) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) }) c.Specify("sets subdecoder runner correctly", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) // Call LogError to appease the angry gomock gods. dRunner.LogError(errors.New("foo")) // Now create a real *dRunner, pass it in, make sure a wrapper // gets handed to the subdecoder. dr := NewDecoderRunner(decoder.Name, decoder, new(PluginGlobals)) decoder.SetDecoderRunner(dr) sub := decoder.Decoders["StartsWithM"] subRunner := sub.(*PayloadRegexDecoder).dRunner c.Expect(subRunner.Name(), gs.Equals, fmt.Sprintf("%s-StartsWithM", decoder.Name)) c.Expect(subRunner.Decoder(), gs.Equals, sub) }) c.Specify("with multiple registered decoders", func() { conf.Subs["StartsWithS"] = make(map[string]interface{}, 0) withS := conf.Subs["StartsWithS"].(map[string]interface{}) withS["type"] = "PayloadRegexDecoder" withS["match_regex"] = "^(?P<TheData>s.*)" withSFields := make(map[string]interface{}, 0) withSFields["StartsWithS"] = "%TheData%" withS["message_fields"] = withSFields conf.Subs["StartsWithM2"] = make(map[string]interface{}, 0) withM2 := conf.Subs["StartsWithM2"].(map[string]interface{}) withM2["type"] = "PayloadRegexDecoder" withM2["match_regex"] = "^(?P<TheData>m.*)" withM2Fields := make(map[string]interface{}, 0) withM2Fields["StartsWithM2"] = "%TheData%" withM2["message_fields"] = withM2Fields conf.Order = append(conf.Order, "StartsWithS", "StartsWithM2") var ok bool // Two more subdecoders means two more LogError calls. dRunner.EXPECT().LogError(gomock.Any()).Times(2) c.Specify("defaults to `first-wins` cascading", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("on a first match condition", func() { regexData := "match first" pack.Message.SetPayload(regexData) _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("and a second match condition", func() { regexData := "second match" pack.Message.SetPayload(regexData) _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("returning an error if they all fail", func() { regexData := "won't match" pack.Message.SetPayload(regexData) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) }) c.Specify("and using `all` cascading", func() { conf.CascadeStrategy = "all" err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("matches multiples when appropriate", func() { regexData := "matches twice" pack.Message.SetPayload(regexData) _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsTrue) }) c.Specify("matches singles when appropriate", func() { regexData := "second match" pack.Message.SetPayload(regexData) _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("returns an error if they all fail", func() { regexData := "won't match" pack.Message.SetPayload(regexData) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) }) }) }) }
func LogstreamerInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() here, _ := os.Getwd() dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir") tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests") c.Expect(tmpErr, gs.Equals, nil) defer func() { tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.IsNil) }() globals := DefaultGlobals() globals.BaseDir = tmpDir config := NewPipelineConfig(globals) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // Specify localhost, but we're not really going to use the network ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) c.Specify("A LogstreamerInput", func() { lsInput := &LogstreamerInput{pConfig: config} lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig) lsiConfig.LogDirectory = dirPath lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?` lsiConfig.Differentiator = []string{"logfile"} lsiConfig.Priority = []string{"^Seq"} lsiConfig.Decoder = "decoder-name" c.Specify("w/ no translation map", func() { err := lsInput.Init(lsiConfig) c.Expect(err, gs.IsNil) c.Expect(len(lsInput.plugins), gs.Equals, 1) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Create pool of packs. numLines := 5 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } c.Specify("reads a log file", func() { // Expect InputRunner calls to get InChan and inject outgoing msgs ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines) // Expect calls to get decoder and decode each message. Since the // decoding is a no-op, the message payload will be the log file // line, unchanged. pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder, "-"+lsiConfig.Decoder) pbcall.Return(mockDecoderRunner, true) decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines) decodeCall.Return(ith.DecodeChan) runOutChan := make(chan error, 1) go func() { err = lsInput.Run(ith.MockInputRunner, ith.MockHelper) runOutChan <- err }() d, _ := time.ParseDuration("5s") timeout := time.After(d) timed := false for x := 0; x < numLines; x++ { select { case <-ith.DecodeChan: case <-timeout: timed = true x += numLines } // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } lsInput.Stop() c.Expect(timed, gs.Equals, false) c.Expect(<-runOutChan, gs.Equals, nil) }) }) c.Specify("with a translation map", func() { lsiConfig.Translation = make(ls.SubmatchTranslationMap) lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap) c.Specify("allows len 1 translation map for 'missing'", func() { lsiConfig.Translation["Seq"]["missing"] = 9999 err := lsInput.Init(lsiConfig) c.Expect(err, gs.IsNil) }) c.Specify("doesn't allow len 1 map for other keys", func() { lsiConfig.Translation["Seq"]["missin"] = 9999 err := lsInput.Init(lsiConfig) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "A translation map with one entry ('Seq') must be specifying a "+ "'missing' key.") }) }) }) }
func UdpInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) c.Specify("A UdpInput", func() { udpInput := UdpInput{} err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr, Decoder: "ProtobufDecoder", ParserType: "message.proto"}) c.Assume(err, gs.IsNil) realListener := (udpInput.listener).(*net.UDPConn) c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr) mbytes, _ := proto.Marshal(ith.Msg) header := &message.Header{} header.SetMessageLength(uint32(len(mbytes))) mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner) mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Name().Return("UdpInput") encCall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "UdpInput-ProtobufDecoder") encCall.Return(ith.Decoder, true) c.Specify("reads a message from the connection and passes it to the decoder", func() { hbytes, _ := proto.Marshal(header) go func() { udpInput.Run(ith.MockInputRunner, ith.MockHelper) }() conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block c.Assume(err, gs.IsNil) buf := encodeMessage(hbytes, mbytes) _, err = conn.Write(buf) c.Assume(err, gs.IsNil) ith.PackSupply <- ith.Pack packRef := <-ith.DecodeChan udpInput.Stop() c.Expect(ith.Pack, gs.Equals, packRef) c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes)) c.Expect(ith.Pack.Decoded, gs.IsFalse) }) }) c.Specify("A UdpInput Multiline input", func() { ith.AddrStr = "localhost:55566" ith.ResolvedAddrStr = "127.0.0.1:55566" udpInput := UdpInput{} err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr, Decoder: "test", ParserType: "token"}) c.Assume(err, gs.IsNil) realListener := (udpInput.listener).(*net.UDPConn) c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr) mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner) mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).Times(2) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(2) ith.MockInputRunner.EXPECT().Name().Return("UdpInput").AnyTimes() encCall := ith.MockHelper.EXPECT().DecoderRunner("test", "UdpInput-test") encCall.Return(ith.Decoder, true) c.Specify("reads two messages from a packet and passes them to the decoder", func() { go func() { udpInput.Run(ith.MockInputRunner, ith.MockHelper) }() conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block c.Assume(err, gs.IsNil) _, err = conn.Write([]byte("message1\nmessage2\n")) c.Assume(err, gs.IsNil) ith.PackSupply <- ith.Pack packRef := <-ith.DecodeChan c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message1\n") ith.PackSupply <- ith.Pack packRef = <-ith.DecodeChan c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message2\n") udpInput.Stop() }) }) }
func StatsToFieldsDecoderSpec(c gospec.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() globals := &GlobalConfigStruct{ PluginChanSize: 5, } config := NewPipelineConfig(globals) c.Specify("A StatsToFieldsDecoder", func() { decoder := new(StatsToFieldsDecoder) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.runner = dRunner pack := NewPipelinePack(config.InputRecycleChan()) mergeStats := func(stats [][]string) string { lines := make([]string, len(stats)) for i, line := range stats { lines[i] = strings.Join(line, " ") } return strings.Join(lines, "\n") } c.Specify("correctly converts stats to fields", func() { stats := [][]string{ {"stat.one", "1", "1380047333"}, {"stat.two", "2", "1380047333"}, {"stat.three", "3", "1380047333"}, {"stat.four", "4", "1380047333"}, {"stat.five", "5", "1380047333"}, } pack.Message.SetPayload(mergeStats(stats)) _, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) for i, stats := range stats { value, ok := pack.Message.GetFieldValue(stats[0]) c.Expect(ok, gs.IsTrue) expected := float64(i + 1) c.Expect(value.(float64), gs.Equals, expected) } value, ok := pack.Message.GetFieldValue("timestamp") c.Expect(ok, gs.IsTrue) expected, err := strconv.ParseInt(stats[0][2], 0, 32) c.Assume(err, gs.IsNil) c.Expect(value.(int64), gs.Equals, expected) }) c.Specify("generates multiple messages for multiple timestamps", func() { stats := [][]string{ {"stat.one", "1", "1380047333"}, {"stat.two", "2", "1380047333"}, {"stat.three", "3", "1380047331"}, {"stat.four", "4", "1380047333"}, {"stat.five", "5", "1380047332"}, } // Prime the pack supply w/ two new packs. dRunner.EXPECT().NewPack().Return(NewPipelinePack(nil)) dRunner.EXPECT().NewPack().Return(NewPipelinePack(nil)) // Decode and check the main pack. pack.Message.SetPayload(mergeStats(stats)) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) value, ok := pack.Message.GetFieldValue("timestamp") c.Expect(ok, gs.IsTrue) expected, err := strconv.ParseInt(stats[0][2], 0, 32) c.Assume(err, gs.IsNil) c.Expect(value.(int64), gs.Equals, expected) // Check the first extra. pack = packs[1] value, ok = pack.Message.GetFieldValue("timestamp") c.Expect(ok, gs.IsTrue) expected, err = strconv.ParseInt(stats[2][2], 0, 32) c.Assume(err, gs.IsNil) c.Expect(value.(int64), gs.Equals, expected) // Check the second extra. pack = packs[2] value, ok = pack.Message.GetFieldValue("timestamp") c.Expect(ok, gs.IsTrue) expected, err = strconv.ParseInt(stats[4][2], 0, 32) c.Assume(err, gs.IsNil) c.Expect(value.(int64), gs.Equals, expected) }) c.Specify("fails w/ invalid timestamp", func() { stats := [][]string{ {"stat.one", "1", "1380047333"}, {"stat.two", "2", "1380047333"}, {"stat.three", "3", "1380047333c"}, {"stat.four", "4", "1380047333"}, {"stat.five", "5", "1380047332"}, } pack.Message.SetPayload(mergeStats(stats)) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err, gs.Not(gs.IsNil)) expected := fmt.Sprintf("invalid timestamp: '%s'", strings.Join(stats[2], " ")) c.Expect(err.Error(), gs.Equals, expected) }) c.Specify("fails w/ invalid value", func() { stats := [][]string{ {"stat.one", "1", "1380047333"}, {"stat.two", "2", "1380047333"}, {"stat.three", "3", "1380047333"}, {"stat.four", "4d", "1380047333"}, {"stat.five", "5", "1380047332"}, } pack.Message.SetPayload(mergeStats(stats)) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err, gs.Not(gs.IsNil)) expected := fmt.Sprintf("invalid value: '%s'", strings.Join(stats[3], " ")) c.Expect(err.Error(), gs.Equals, expected) }) }) }
func HttpListenInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) httpListenInput := HttpListenInput{} ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) startInput := func() { go func() { httpListenInput.Run(ith.MockInputRunner, ith.MockHelper) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) config := httpListenInput.ConfigStruct().(*HttpListenInputConfig) config.Decoder = "PayloadJsonDecoder" mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) dRunnerInChan := make(chan *PipelinePack, 1) c.Specify("A HttpListenInput", func() { mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Name().Return("HttpListenInput").Times(2) ith.MockHelper.EXPECT().DecoderRunner("PayloadJsonDecoder", "HttpListenInput-PayloadJsonDecoder").Return(mockDecoderRunner, true) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) startedChan := make(chan bool, 1) defer close(startedChan) ts := httptest.NewUnstartedServer(nil) httpListenInput.starterFunc = func(hli *HttpListenInput) error { ts.Start() startedChan <- true return nil } c.Specify("Adds query parameters to the message pack as fields", func() { err := httpListenInput.Init(config) ts.Config = httpListenInput.server c.Assume(err, gs.IsNil) startInput() ith.PackSupply <- ith.Pack <-startedChan resp, err := http.Get(ts.URL + "/?test=Hello%20World") c.Assume(err, gs.IsNil) resp.Body.Close() c.Assume(resp.StatusCode, gs.Equals, 200) pack := <-dRunnerInChan fieldValue, ok := pack.Message.GetFieldValue("test") c.Assume(ok, gs.IsTrue) c.Expect(fieldValue, gs.Equals, "Hello World") }) c.Specify("Add custom headers", func() { config.Headers = http.Header{ "One": []string{"two", "three"}, "Four": []string{"five", "six", "seven"}, } err := httpListenInput.Init(config) ts.Config = httpListenInput.server c.Assume(err, gs.IsNil) startInput() ith.PackSupply <- ith.Pack <-startedChan resp, err := http.Get(ts.URL) c.Assume(err, gs.IsNil) resp.Body.Close() c.Assume(resp.StatusCode, gs.Equals, 200) <-dRunnerInChan // Verify headers are there eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"]) c.Expect(eq, gs.IsTrue) eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"]) c.Expect(eq, gs.IsTrue) }) ts.Close() httpListenInput.Stop() }) }
func LogfileInputSpec0(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // Specify localhost, but we're not really going to use the network ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) c.Specify("A LogFileInput", func() { tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-") c.Expect(tmpErr, gs.Equals, nil) origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.IsNil) }() lfInput := new(LogfileInput) lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig) lfiConfig.SeekJournalName = "test-seekjournal" lfiConfig.LogFile = "../testsupport/test-zeus.log" lfiConfig.Logger = "zeus" lfiConfig.UseSeekJournal = true lfiConfig.Decoder = "decoder-name" lfiConfig.DiscoverInterval = 1 lfiConfig.StatInterval = 1 err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Create pool of packs. numLines := 95 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } c.Specify("reads a log file", func() { // Expect InputRunner calls to get InChan and inject outgoing msgs ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines) // Expect calls to get decoder and decode each message. Since the // decoding is a no-op, the message payload will be the log file // line, unchanged. ith.MockInputRunner.EXPECT().Name().Return("LogfileInput") pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder) pbcall.Return(mockDecoderRunner, true) decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines) decodeCall.Return(ith.DecodeChan) go func() { err = lfInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() for x := 0; x < numLines; x++ { _ = <-ith.DecodeChan // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } lfInput.Stop() fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile) c.Expect(err, gs.IsNil) fileStr := string(fileBytes) lines := strings.Split(fileStr, "\n") for i, line := range lines { if line == "" { continue } c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n") c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus") } // Wait for the file update to hit the disk; better suggestions are welcome runtime.Gosched() time.Sleep(time.Millisecond * 250) journalData := []byte(`{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_len":300,"last_start":28650,"seek":28950}`) journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName)) c.Expect(err, gs.IsNil) c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0) }) c.Specify("uses the filename as the default logger name", func() { lfInput := new(LogfileInput) lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig) lfiConfig.LogFile = "../testsupport/test-zeus.log" lfiConfig.DiscoverInterval = 1 lfiConfig.StatInterval = 1 err := lfInput.Init(lfiConfig) c.Expect(err, gs.Equals, nil) c.Expect(lfInput.Monitor.logger_ident, gs.Equals, lfiConfig.LogFile) }) }) c.Specify("A Regex LogFileInput", func() { tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-") c.Expect(tmpErr, gs.Equals, nil) origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() lfInput := new(LogfileInput) lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig) lfiConfig.SeekJournalName = "regex-seekjournal" lfiConfig.LogFile = "../testsupport/test-zeus.log" lfiConfig.Logger = "zeus" lfiConfig.ParserType = "regexp" lfiConfig.UseSeekJournal = true lfiConfig.Decoder = "decoder-name" lfiConfig.DiscoverInterval = 1 lfiConfig.StatInterval = 1 mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Create pool of packs. numLines := 95 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } c.Specify("doesn't whack default RegexpParser delimiter", func() { err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) parser := lfInput.Monitor.parser.(*RegexpParser) buf := bytes.NewBuffer([]byte("split\nhere")) n, r, err := parser.Parse(buf) c.Expect(n, gs.Equals, len("split\n")) c.Expect(string(r), gs.Equals, "split") c.Expect(err, gs.IsNil) }) c.Specify("keeps captures in the record text", func() { lfiConfig.Delimiter = "(\n)" err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) parser := lfInput.Monitor.parser.(*RegexpParser) buf := bytes.NewBuffer([]byte("split\nhere")) n, r, err := parser.Parse(buf) c.Expect(n, gs.Equals, len("split\n")) c.Expect(string(r), gs.Equals, "split\n") c.Expect(err, gs.IsNil) }) c.Specify("reads a log file", func() { lfiConfig.Delimiter = "(\n)" err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) // Expect InputRunner calls to get InChan and inject outgoing msgs ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines) // Expect calls to get decoder and decode each message. Since the // decoding is a no-op, the message payload will be the log file // line, unchanged. ith.MockInputRunner.EXPECT().Name().Return("LogfileInput") pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder) pbcall.Return(mockDecoderRunner, true) decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines) decodeCall.Return(ith.DecodeChan) go func() { err = lfInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() for x := 0; x < numLines; x++ { _ = <-ith.DecodeChan // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } lfInput.Stop() fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile) c.Expect(err, gs.IsNil) fileStr := string(fileBytes) lines := strings.Split(fileStr, "\n") for i, line := range lines { if line == "" { continue } c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n") c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus") } // Wait for the file update to hit the disk; better suggestions are welcome runtime.Gosched() time.Sleep(time.Millisecond * 250) journalData := []byte(`{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_len":300,"last_start":28650,"seek":28950}`) journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName)) c.Expect(err, gs.IsNil) c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0) }) }) c.Specify("A Regex Multiline LogFileInput", func() { tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-") c.Expect(tmpErr, gs.Equals, nil) origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() lfInput := new(LogfileInput) lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig) lfiConfig.SeekJournalName = "multiline-seekjournal" lfiConfig.LogFile = "../testsupport/multiline.log" lfiConfig.Logger = "multiline" lfiConfig.ParserType = "regexp" lfiConfig.Delimiter = "\n(\\d{4}-\\d{2}-\\d{2})" lfiConfig.DelimiterLocation = "start" lfiConfig.UseSeekJournal = true lfiConfig.Decoder = "decoder-name" lfiConfig.DiscoverInterval = 1 lfiConfig.StatInterval = 1 err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Create pool of packs. numLines := 4 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } c.Specify("reads a log file", func() { // Expect InputRunner calls to get InChan and inject outgoing msgs ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines) // Expect calls to get decoder and decode each message. Since the // decoding is a no-op, the message payload will be the log file // line, unchanged. ith.MockInputRunner.EXPECT().Name().Return("LogfileInput") pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder) pbcall.Return(mockDecoderRunner, true) decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines) decodeCall.Return(ith.DecodeChan) go func() { err = lfInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() for x := 0; x < numLines; x++ { _ = <-ith.DecodeChan // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } lfInput.Stop() lines := []string{ "2012-07-13 18:48:01 debug readSocket()", "2012-07-13 18:48:21 info Processing queue id 3496 -> subm id 2817 from site ms", "2012-07-13 18:48:25 debug line0\nline1\nline2", "2012-07-13 18:48:26 debug readSocket()", } for i, line := range lines { c.Expect(packs[i].Message.GetPayload(), gs.Equals, line) c.Expect(packs[i].Message.GetLogger(), gs.Equals, "multiline") } // Wait for the file update to hit the disk; better suggestions are welcome runtime.Gosched() time.Sleep(time.Millisecond * 250) journalData := []byte(`{"last_hash":"39e4c3e6e9c88a794b3e7c91c155682c34cf1a4a","last_len":41,"last_start":172,"seek":214}`) journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName)) c.Expect(err, gs.IsNil) c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0) }) }) c.Specify("A message.proto LogFileInput", func() { tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-") c.Expect(tmpErr, gs.Equals, nil) origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() lfInput := new(LogfileInput) lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig) lfiConfig.SeekJournalName = "protobuf-seekjournal" lfiConfig.LogFile = "../testsupport/protobuf.log" lfiConfig.ParserType = "message.proto" lfiConfig.UseSeekJournal = true lfiConfig.Decoder = "decoder-name" lfiConfig.DiscoverInterval = 1 lfiConfig.StatInterval = 1 err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Create pool of packs. numLines := 7 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } c.Specify("reads a log file", func() { // Expect InputRunner calls to get InChan and decode outgoing msgs ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines) ith.MockInputRunner.EXPECT().Name().Return("LogfileInput") pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder) pbcall.Return(mockDecoderRunner, true) decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines) decodeCall.Return(ith.DecodeChan) go func() { err = lfInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() for x := 0; x < numLines; x++ { _ = <-ith.DecodeChan // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } lfInput.Stop() lines := []int{36230, 41368, 42310, 41343, 37171, 56727, 46492} for i, line := range lines { c.Expect(len(packs[i].MsgBytes), gs.Equals, line) err = proto.Unmarshal(packs[i].MsgBytes, packs[i].Message) c.Expect(err, gs.IsNil) c.Expect(packs[i].Message.GetType(), gs.Equals, "hekabench") } // Wait for the file update to hit the disk; better suggestions are welcome runtime.Gosched() time.Sleep(time.Millisecond * 250) journalData := []byte(`{"last_hash":"f67dc6bbbbb6a91b59e661b6170de50c96eab100","last_len":46499,"last_start":255191,"seek":301690}`) journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName)) c.Expect(err, gs.IsNil) c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0) }) }) c.Specify("A message.proto LogFileInput no decoder", func() { tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-") c.Expect(tmpErr, gs.Equals, nil) origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() lfInput := new(LogfileInput) lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig) lfiConfig.SeekJournalName = "protobuf-seekjournal" lfiConfig.LogFile = "../testsupport/protobuf.log" lfiConfig.ParserType = "message.proto" lfiConfig.UseSeekJournal = true lfiConfig.Decoder = "" lfiConfig.DiscoverInterval = 1 lfiConfig.StatInterval = 1 err := lfInput.Init(lfiConfig) c.Expect(err, gs.Not(gs.IsNil)) }) }
func MultiDecoderSpec(c gospec.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) // initializes Globals() c.Specify("A MultiDecoder", func() { decoder := new(MultiDecoder) decoder.SetName("MyMultiDecoder") decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*MultiDecoderConfig) supply := make(chan *PipelinePack, 1) pack := NewPipelinePack(supply) rDecoder0 := new(PayloadRegexDecoder) rDecoder0.Match, _ = regexp.Compile("^(?P<TheData>m.*)") rDecoder0.MessageFields = MessageTemplate{ "StartsWithM": "%TheData%", } rDecoder0.logErrors = true wrapper0 := NewPluginWrapper("StartsWithM", pConfig) wrapper0.CreateWithError = func() (interface{}, error) { return rDecoder0, nil } pConfig.DecoderWrappers["StartsWithM"] = wrapper0 conf.Subs = []string{"StartsWithM"} errMsg := "All subdecoders failed." dRunner := pipelinemock.NewMockDecoderRunner(ctrl) // An error will be spit out b/c there's no real *dRunner in there; // doesn't impact the tests. dRunner.EXPECT().LogError(gomock.Any()) c.Specify("decodes simple messages", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "matching text" pack.Message.SetPayload(regex_data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) value, ok := pack.Message.GetFieldValue("StartsWithM") c.Assume(ok, gs.IsTrue) c.Expect(value, gs.Equals, regex_data) }) c.Specify("returns an error if all decoders fail", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "non-matching text" pack.Message.SetPayload(regex_data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) }) c.Specify("logs subdecoder failures when configured to do so", func() { conf.LogSubErrors = true err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "non-matching text" pack.Message.SetPayload(regex_data) // Expect that we log an error for undecoded message. dRunner.EXPECT().LogError(fmt.Errorf( "Subdecoder 'StartsWithM' decode error: No match: %s", regex_data)).AnyTimes() packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) }) c.Specify("sets subdecoder runner correctly", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) // Call LogError to appease the angry gomock gods. dRunner.LogError(errors.New("foo")) // Now create a real *dRunner, pass it in, make sure a wrapper // gets handed to the subdecoder. dr := NewDecoderRunner(decoder.Name, decoder, new(PluginGlobals), 10) decoder.SetDecoderRunner(dr) sub := decoder.Decoders[0] subRunner := sub.(*PayloadRegexDecoder).dRunner c.Expect(subRunner.Name(), gs.Equals, fmt.Sprintf("%s-StartsWithM", decoder.Name)) c.Expect(subRunner.Decoder(), gs.Equals, sub) }) c.Specify("with multiple registered decoders", func() { rDecoder1 := new(PayloadRegexDecoder) rDecoder1.Match, _ = regexp.Compile("^(?P<TheData>s.*)") rDecoder1.MessageFields = MessageTemplate{ "StartsWithS": "%TheData%", } wrapper1 := NewPluginWrapper("StartsWithS", pConfig) wrapper1.CreateWithError = func() (interface{}, error) { return rDecoder1, nil } pConfig.DecoderWrappers["StartsWithS"] = wrapper1 rDecoder2 := new(PayloadRegexDecoder) rDecoder2.Match, _ = regexp.Compile("^(?P<TheData>m.*)") rDecoder2.MessageFields = MessageTemplate{ "StartsWithM2": "%TheData%", } wrapper2 := NewPluginWrapper("StartsWithM2", pConfig) wrapper2.CreateWithError = func() (interface{}, error) { return rDecoder2, nil } pConfig.DecoderWrappers["StartsWithM2"] = wrapper2 conf.Subs = append(conf.Subs, "StartsWithS", "StartsWithM2") var ok bool // Two more subdecoders means two more LogError calls. dRunner.EXPECT().LogError(gomock.Any()).Times(2) c.Specify("defaults to `first-wins` cascading", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("on a first match condition", func() { pack.Message.SetPayload("match first") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("and a second match condition", func() { pack.Message.SetPayload("second match") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("returning an error if they all fail", func() { pack.Message.SetPayload("won't match") packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) }) c.Specify("and using `all` cascading", func() { conf.CascadeStrategy = "all" err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("matches multiples when appropriate", func() { pack.Message.SetPayload("matches twice") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsTrue) }) c.Specify("matches singles when appropriate", func() { pack.Message.SetPayload("second match") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("returns an error if they all fail", func() { pack.Message.SetPayload("won't match") packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) }) }) }) c.Specify("A MultiDecoder w/ MultiOutput", func() { decoder := new(MultiDecoder) decoder.SetName("MyMultiDecoder") decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*MultiDecoderConfig) conf.CascadeStrategy = "all" supply := make(chan *PipelinePack, 10) pack := NewPipelinePack(supply) sub0 := &MultiOutputDecoder{supply} sub1 := &MultiOutputDecoder{supply} sub2 := &MultiOutputDecoder{supply} wrapper0 := NewPluginWrapper("sub0", pConfig) wrapper0.CreateWithError = func() (interface{}, error) { return sub0, nil } pConfig.DecoderWrappers["sub0"] = wrapper0 wrapper1 := NewPluginWrapper("sub1", pConfig) wrapper1.CreateWithError = func() (interface{}, error) { return sub1, nil } pConfig.DecoderWrappers["sub1"] = wrapper1 wrapper2 := NewPluginWrapper("sub2", pConfig) wrapper2.CreateWithError = func() (interface{}, error) { return sub2, nil } pConfig.DecoderWrappers["sub2"] = wrapper2 conf.Subs = []string{"sub0", "sub1", "sub2"} dRunner := pipelinemock.NewMockDecoderRunner(ctrl) c.Specify("always tries to decode all packs", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 15) }) }) }
func LogfileInputSpec1(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-") c.Expect(tmpErr, gs.Equals, nil) origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() journalName := "test-seekjournal" journalDir := filepath.Join(tmpDir, "seekjournal") tmpErr = os.MkdirAll(journalDir, 0770) c.Expect(tmpErr, gs.Equals, nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // Specify localhost, but we're not really going to use the network ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) c.Specify("A LogfileInput", func() { c.Specify("save the seek position of the last complete logline", func() { lfInput, lfiConfig := createIncompleteLogfileInput(journalName) // Initialize the input test helper err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) dName := "decoder-name" lfInput.decoderName = dName mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Create pool of packs. numLines := 4 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } // Expect InputRunner calls to get InChan and inject outgoing msgs ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines) // Expect calls to get decoder and decode each message. Since the // decoding is a no-op, the message payload will be the log file // line, unchanged. ith.MockInputRunner.EXPECT().Name().Return("LogfileInput") pbcall := ith.MockHelper.EXPECT().DecoderRunner(dName, "LogfileInput-"+dName) pbcall.Return(mockDecoderRunner, true) decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines) decodeCall.Return(ith.DecodeChan) go func() { err = lfInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() for x := 0; x < numLines; x++ { _ = <-ith.DecodeChan // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } newFM := new(FileMonitor) newFM.Init(lfiConfig) c.Expect(err, gs.Equals, nil) fbytes, _ := json.Marshal(lfInput.Monitor) // Check that the persisted hashcode is from the last // complete log line expected_lastline := `10.1.1.4 plinko-565.byzantium.mozilla.com user3 [15/Mar/2013:12:20:27 -0700] "GET /1.1/user3/storage/passwords?newer=1356237662.44&full=1 HTTP/1.1" 200 1396 "-" "Firefox/20.0.1 FxSync/1.22.0.201304.desktop" "-" "ssl: SSL_RSA_WITH_RC4_128_SHA, version=TLSv1, bits=128" node_s:0.047167 req_s:0.047167 retries:0 req_b:446 "c_l:-"` + "\n" c.Expect((strings.IndexAny(string(fbytes), sha1_hexdigest(expected_lastline)) > -1), gs.IsTrue) json.Unmarshal(fbytes, &newFM) if runtime.GOOS == "windows" { c.Expect(newFM.seek, gs.Equals, int64(1253)) } else { c.Expect(newFM.seek, gs.Equals, int64(1249)) } lfInput.Stop() }) }) c.Specify("A LogfileDirectoryManagerInput", func() { c.Specify("empty file name", func() { var err error ldm := new(LogfileDirectoryManagerInput) conf := ldm.ConfigStruct().(*LogfileInputConfig) conf.LogFile = "" err = ldm.Init(conf) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "A logfile name must be specified.") }) c.Specify("glob in file name", func() { var err error ldm := new(LogfileDirectoryManagerInput) conf := ldm.ConfigStruct().(*LogfileInputConfig) conf.LogFile = "../testsupport/*.log" err = ldm.Init(conf) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "Globs are not allowed in the file name: *.log") }) // Note: Testing the actual functionality (spinning up new plugins within Heka) // is a manual process. }) }
func NsqInputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := pipeline.NewPipelineConfig(nil) var wg sync.WaitGroup errChan := make(chan error, 1) retPackChan := make(chan *pipeline.PipelinePack, 1) defer close(errChan) defer close(retPackChan) c.Specify("A nsq input", func() { input := new(NsqInput) ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) config := input.ConfigStruct().(*NsqInputConfig) config.Topic = "test_topic" config.Channel = "test_channel" startInput := func() { wg.Add(1) go func() { errChan <- input.Run(ith.MockInputRunner, ith.MockHelper) wg.Done() }() } var mockConsumer *MockConsumer input.newConsumer = func(topic, channel string, config *nsq.Config) (Consumer, error) { mockConsumer, _ = NewMockConsumer(topic, channel, config) return mockConsumer, nil } ith.Pack = pipeline.NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *pipeline.PipelinePack, 1) ith.PackSupply <- ith.Pack inputName := "NsqInput" ith.MockInputRunner.EXPECT().Name().Return(inputName).AnyTimes() ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) c.Specify("that is started", func() { input.SetPipelineConfig(pConfig) c.Specify("gets messages its subscribed to", func() { c.Specify("injects messages into the pipeline when not configured with a decoder", func() { ith.MockInputRunner.EXPECT().Inject(ith.Pack).Do(func(p *pipeline.PipelinePack) { retPackChan <- p }).AnyTimes() err := input.Init(config) c.Expect(err, gs.IsNil) c.Expect(input.DecoderName, gs.Equals, "") }) c.Specify("injects messages into the decoder when configured", func() { decoderName := "ScribbleDecoder" config.DecoderName = decoderName decoder := new(plugins.ScribbleDecoder) decoder.Init(&plugins.ScribbleDecoderConfig{}) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) mockDecoderRunner.EXPECT().InChan().Return(retPackChan) ith.MockHelper.EXPECT().DecoderRunner(decoderName, fmt.Sprintf("%s-%s", inputName, decoderName), ).Return(mockDecoderRunner, true) err := input.Init(config) c.Expect(err, gs.IsNil) c.Expect(input.DecoderName, gs.Equals, decoderName) }) startInput() // Should get two finished conn calls since we connect // to lookupds and nsqds c.Expect(<-mockConsumer.finishedConn, gs.IsTrue) c.Expect(<-mockConsumer.finishedConn, gs.IsTrue) id := nsq.MessageID{} body := []byte("test") msg := nsq.NewMessage(id, body) for _, handler := range mockConsumer.handlers { handler.HandleMessage(msg) } p := <-retPackChan c.Expect(p.Message.GetPayload(), gs.Equals, "test") }) input.Stop() wg.Wait() c.Expect(<-errChan, gs.IsNil) }) }) }
func FileMonitorSpec(c gs.Context) { config := NewPipelineConfig(nil) tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-") c.Expect(tmpErr, gs.Equals, nil) origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() journalName := "test-seekjournal" journalDir := filepath.Join(tmpDir, "seekjournals") tmpErr = os.MkdirAll(journalDir, 0770) c.Expect(tmpErr, gs.Equals, nil) journalPath := filepath.Join(journalDir, journalName) t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // Specify localhost, but we're not really going to use the network ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) c.Specify("saved last read position", func() { c.Specify("without a previous journal", func() { ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() lfInput, lfiConfig := createLogfileInput(journalName) // Initialize the input test helper err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) dName := "decoder-name" lfInput.decoderName = dName mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Create pool of packs. numLines := 95 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } // Expect InputRunner calls to get InChan and inject outgoing msgs ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines) // Expect calls to get decoder and decode each message. Since the // decoding is a no-op, the message payload will be the log file // line, unchanged. ith.MockInputRunner.EXPECT().Name().Return("FileMonitor") ith.MockHelper.EXPECT().DecoderRunner(dName, "FileMonitor-"+dName).Return(mockDecoderRunner, true) decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines) decodeCall.Return(ith.DecodeChan) go func() { err = lfInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() for x := 0; x < numLines; x++ { _ = <-ith.DecodeChan // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } newFM := new(FileMonitor) newFM.Init(lfiConfig) c.Expect(err, gs.Equals, nil) fbytes, _ := json.Marshal(lfInput.Monitor) json.Unmarshal(fbytes, &newFM) c.Expect(newFM.seek, gs.Equals, int64(28950)) lfInput.Stop() }) c.Specify("with a previous journal initializes with a seek value", func() { lfInput, lfiConfig := createLogfileInput(journalName) journalData := `{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_start":28650,"last_len":300,"seek":28950}` journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660) c.Expect(journalErr, gs.Equals, nil) journal.WriteString(journalData) journal.Close() err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) c.Expect(err, gs.IsNil) // # bytes should be set to what's in the journal data c.Expect(lfInput.Monitor.seek, gs.Equals, int64(28950)) }) c.Specify("resets last read position to 0 if hash doesn't match", func() { lfInput, lfiConfig := createLogfileInput(journalName) lfiConfig.ResumeFromStart = true journalData := `{"last_hash":"xxxxx","last_start":28650,"last_len":300,"seek":28950}` journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660) c.Expect(journalErr, gs.Equals, nil) journal.WriteString(journalData) journal.Close() err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) // # bytes should be set to what's in the journal data c.Expect(lfInput.Monitor.seek, gs.Equals, int64(0)) }) c.Specify("resets last read position to end of file if hash doesn't match", func() { lfInput, lfiConfig := createLogfileInput(journalName) lfiConfig.ResumeFromStart = false journalData := `{"last_hash":"xxxxx","last_start":28650,"last_len":300,"seek":28950}` journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660) c.Expect(journalErr, gs.Equals, nil) journal.WriteString(journalData) journal.Close() err := lfInput.Init(lfiConfig) c.Expect(err, gs.IsNil) // # bytes should be set to what's in the journal data c.Expect(lfInput.Monitor.seek, gs.Equals, int64(28950)) }) }) c.Specify("filemonitor generates expected journal path", func() { lfInput := new(LogfileInput) lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig) lfiConfig.LogFile = filepath.Join("..", "testsupport", "test-zeus.log") lfiConfig.DiscoverInterval = 5 lfiConfig.StatInterval = 5 err := lfInput.Init(lfiConfig) c.Expect(err, gs.Equals, nil) clean := filepath.Join(journalDir, "___testsupport_test-zeus_log") c.Expect(lfInput.Monitor.seekJournalPath, gs.Equals, clean) }) }
func ProcessInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // Specify localhost, but we're not really going to use the network ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) c.Specify("A ProcessInput", func() { pInput := ProcessInput{} ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes() ith.MockInputRunner.EXPECT().Name().Return("logger").AnyTimes() enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "logger-RegexpDecoder").AnyTimes() enccall.Return(ith.Decoder, true) mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner) mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).AnyTimes() config := pInput.ConfigStruct().(*ProcessInputConfig) config.Command = make(map[string]cmdConfig) pConfig := NewPipelineConfig(nil) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) c.Specify("reads a message from ProcessInput", func() { pInput.SetName("SimpleTest") config.Decoder = "RegexpDecoder" config.ParserType = "token" config.Delimiter = "|" // Note that no working directory is explicitly specified config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_TEST1_CMD, Args: PROCESSINPUT_TEST1_CMD_ARGS} err := pInput.Init(config) c.Assume(err, gs.IsNil) go func() { pInput.Run(ith.MockInputRunner, ith.MockHelper) }() tickChan <- time.Now() expected_payloads := PROCESSINPUT_TEST1_OUTPUT actual_payloads := []string{} for x := 0; x < 4; x++ { ith.PackSupply <- ith.Pack packRef := <-ith.DecodeChan c.Expect(ith.Pack, gs.Equals, packRef) actual_payloads = append(actual_payloads, *packRef.Message.Payload) fPInputName := *packRef.Message.FindFirstField("ProcessInputName") c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout") // Free up the scheduler runtime.Gosched() } for x := 0; x < 4; x++ { c.Expect(expected_payloads[x], gs.Equals, actual_payloads[x]) } pInput.Stop() }) c.Specify("handles bad arguments", func() { pInput.SetName("BadArgs") config.ParseStdout = false config.ParseStderr = true config.Decoder = "RegexpDecoder" config.ParserType = "token" config.Delimiter = "|" // Note that no working directory is explicitly specified config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS} err := pInput.Init(config) c.Assume(err, gs.IsNil) expected_err := fmt.Errorf("BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]") ith.MockInputRunner.EXPECT().LogError(expected_err) go func() { pInput.Run(ith.MockInputRunner, ith.MockHelper) }() tickChan <- time.Now() ith.PackSupply <- ith.Pack <-ith.DecodeChan runtime.Gosched() pInput.Stop() }) c.Specify("can pipe multiple commands together", func() { pInput.SetName("PipedCmd") config.Decoder = "RegexpDecoder" config.ParserType = "token" // Overload the delimiter config.Delimiter = " " // Note that no working directory is explicitly specified config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD1, Args: PROCESSINPUT_PIPE_CMD1_ARGS} config.Command["1"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD2, Args: PROCESSINPUT_PIPE_CMD2_ARGS} err := pInput.Init(config) c.Assume(err, gs.IsNil) go func() { pInput.Run(ith.MockInputRunner, ith.MockHelper) }() tickChan <- time.Now() expected_payloads := PROCESSINPUT_PIPE_OUTPUT actual_payloads := []string{} for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ { ith.PackSupply <- ith.Pack packRef := <-ith.DecodeChan c.Expect(ith.Pack, gs.Equals, packRef) actual_payloads = append(actual_payloads, *packRef.Message.Payload) fPInputName := *packRef.Message.FindFirstField("ProcessInputName") c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout") // Free up the scheduler runtime.Gosched() } for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ { c.Expect(fmt.Sprintf("[%d] [%s] [%x]", len(actual_payloads[x]), actual_payloads[x], actual_payloads[x]), gs.Equals, fmt.Sprintf("[%d] [%s] [%x]", len(expected_payloads[x]), expected_payloads[x], expected_payloads[x])) } pInput.Stop() }) }) }
func PayloadDecodersSpec(c gospec.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() c.Specify("A PayloadXmlDecoder", func() { decoder := new(PayloadXmlDecoder) conf := decoder.ConfigStruct().(*PayloadXmlDecoderConfig) supply := make(chan *PipelinePack, 1) pack := NewPipelinePack(supply) c.Specify("decodes simple messages", func() { xml_data := `<library> <!-- Great book. --> <book id="b0836217462" available="true"> <isbn>0836217462</isbn> <title lang="en">Being a Dog Is a Full-Time Job</title> <quote>I'd dog paddle the deepest ocean.</quote> <author id="CMS"> <?echo "go rocks"?> <name>Charles M Schulz</name> <born>1922-11-26</born> <dead>2000-02-12</dead> </author> <character id="PP"> <name>Peppermint Patty</name> <born>1966-08-22</born> <qualificati>bold, brash and tomboyish</qualificati> </character> <character id="Snoopy"> <name>Snoopy</name> <born>1950-10-04</born> <qualificati>extroverted beagle</qualificati> </character> </book> </library>` conf.XPathMapConfig = map[string]string{"Isbn": "library/*/isbn", "Name": "/library/book/character[born='1950-10-04']/name", "Patty": "/library/book//node()[@id='PP']/name", "Title": "//book[author/@id='CMS']/title", "Comment": "/library/book/preceding::comment()", } conf.MessageFields = MessageTemplate{ "Isbn": "%Isbn%", "Name": "%Name%", "Patty": "%Patty%", "Title": "%Title%", "Comment": "%Comment%", } err := decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload(xml_data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) var isbn, name, patty, title, comment interface{} var ok bool isbn, ok = pack.Message.GetFieldValue("Isbn") c.Expect(ok, gs.Equals, true) name, ok = pack.Message.GetFieldValue("Name") c.Expect(ok, gs.Equals, true) patty, ok = pack.Message.GetFieldValue("Patty") c.Expect(ok, gs.Equals, true) title, ok = pack.Message.GetFieldValue("Title") c.Expect(ok, gs.Equals, true) comment, ok = pack.Message.GetFieldValue("Comment") c.Expect(ok, gs.Equals, true) c.Expect(isbn, gs.Equals, "0836217462") c.Expect(name, gs.Equals, "Snoopy") c.Expect(patty, gs.Equals, "Peppermint Patty") c.Expect(title, gs.Equals, "Being a Dog Is a Full-Time Job") c.Expect(comment, gs.Equals, " Great book. ") }) }) c.Specify("A PayloadRegexDecoder", func() { decoder := new(PayloadRegexDecoder) conf := decoder.ConfigStruct().(*PayloadRegexDecoderConfig) supply := make(chan *PipelinePack, 1) pack := NewPipelinePack(supply) conf.TimestampLayout = "02/Jan/2006:15:04:05 -0700" c.Specify("non capture regex", func() { conf.MatchRegex = `\d+` err := decoder.Init(conf) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "PayloadRegexDecoder regex must contain capture groups") }) c.Specify("invalid regex", func() { conf.MatchRegex = `\mtest` err := decoder.Init(conf) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "PayloadRegexDecoder: error parsing regexp: invalid escape sequence: `\\m`") }) c.Specify("reading an apache timestamp", func() { conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]` err := decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload("[18/Apr/2013:14:00:28 -0700]") _, err = decoder.Decode(pack) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1366318828000000000)) pack.Zero() }) c.Specify("logs invalid messages", func() { conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]` err := decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload("invalid payload") _, err = decoder.Decode(pack) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "No match: invalid payload") pack.Zero() }) c.Specify("ignores invalid messages when log_errors is disabled", func() { conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]` conf.LogErrors = false err := decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload("invalid payload") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) pack.Zero() }) c.Specify("uses kitchen timestamp", func() { conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]` err := decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload("[5:16PM]") now := time.Now() cur_date := time.Date(now.Year(), now.Month(), now.Day(), 17, 16, 0, 0, time.UTC) _, err = decoder.Decode(pack) c.Expect(pack.Message.GetTimestamp(), gs.Equals, cur_date.UnixNano()) pack.Zero() }) c.Specify("adjusts timestamps as specified", func() { conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]` conf.TimestampLayout = "02/Jan/2006:15:04:05" conf.TimestampLocation = "America/Los_Angeles" timeStr := "18/Apr/2013:14:00:28" loc, err := time.LoadLocation(conf.TimestampLocation) c.Assume(err, gs.IsNil) expectedLocal, err := time.ParseInLocation(conf.TimestampLayout, timeStr, loc) c.Assume(err, gs.IsNil) err = decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload("[" + timeStr + "]") _, err = decoder.Decode(pack) c.Expect(pack.Message.GetTimestamp(), gs.Equals, expectedLocal.UnixNano()) pack.Zero() }) c.Specify("apply representation metadata to a captured field", func() { value := "0.23" payload := "header" conf.MatchRegex = `(?P<ResponseTime>\d+\.\d+)` conf.MessageFields = MessageTemplate{ "ResponseTime|s": "%ResponseTime%", "Payload|s": "%ResponseTime%", "Payload": payload, } err := decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload(value) _, err = decoder.Decode(pack) f := pack.Message.FindFirstField("ResponseTime") c.Expect(f, gs.Not(gs.IsNil)) c.Expect(f.GetValue(), gs.Equals, value) c.Expect(f.GetRepresentation(), gs.Equals, "s") f = pack.Message.FindFirstField("Payload") c.Expect(f, gs.Not(gs.IsNil)) c.Expect(f.GetValue(), gs.Equals, value) c.Expect(f.GetRepresentation(), gs.Equals, "s") c.Expect(pack.Message.GetPayload(), gs.Equals, payload) pack.Zero() }) c.Specify("reading test-zeus.log", func() { conf.MatchRegex = `(?P<Ip>([0-9]{1,3}\.){3}[0-9]{1,3}) (?P<Hostname>(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])) (?P<User>\w+) \[(?P<Timestamp>[^\]]+)\] \"(?P<Verb>[A-X]+) (?P<Request>\/\S*) HTTP\/(?P<Httpversion>\d\.\d)\" (?P<Response>\d{3}) (?P<Bytes>\d+)` conf.MessageFields = MessageTemplate{ "hostname": "%Hostname%", "ip": "%Ip%", "response": "%Response%", } err := decoder.Init(conf) c.Assume(err, gs.IsNil) filePath := "../testsupport/test-zeus.log" fileBytes, err := ioutil.ReadFile(filePath) c.Assume(err, gs.IsNil) fileStr := string(fileBytes) lines := strings.Split(fileStr, "\n") containsFieldValue := func(str, fieldName string, msg *message.Message) bool { raw, ok := msg.GetFieldValue(fieldName) if !ok { return false } value := raw.(string) return strings.Contains(str, value) } c.Specify("extracts capture data and puts it in the message fields", func() { var misses int for _, line := range lines { if strings.TrimSpace(line) == "" { continue } pack.Message.SetPayload(line) _, err = decoder.Decode(pack) if err != nil { misses++ continue } c.Expect(containsFieldValue(line, "hostname", pack.Message), gs.IsTrue) c.Expect(containsFieldValue(line, "ip", pack.Message), gs.IsTrue) c.Expect(containsFieldValue(line, "response", pack.Message), gs.IsTrue) pack.Zero() } c.Expect(misses, gs.Equals, 3) }) }) c.Specify("reading test-severity.log", func() { conf.MatchRegex = `severity: (?P<Severity>[a-zA-Z]+)` conf.SeverityMap = map[string]int32{ "emergency": 0, "alert": 1, "critical": 2, "error": 3, "warning": 4, "notice": 5, "info": 6, "debug": 7, } reverseMap := make(map[int32]string) for str, i := range conf.SeverityMap { reverseMap[i] = str } err := decoder.Init(conf) c.Assume(err, gs.IsNil) dRunner := pipelinemock.NewMockDecoderRunner(ctrl) decoder.SetDecoderRunner(dRunner) filePath := "../testsupport/test-severity.log" fileBytes, err := ioutil.ReadFile(filePath) c.Assume(err, gs.IsNil) fileStr := string(fileBytes) lines := strings.Split(fileStr, "\n") c.Specify("sets message severity based on SeverityMap", func() { err := errors.New("Don't recognize severity: 'BOGUS'") dRunner.EXPECT().LogError(err) for _, line := range lines { if strings.TrimSpace(line) == "" { continue } pack.Message.SetPayload(line) _, err = decoder.Decode(pack) if err != nil { fmt.Println(line) } c.Expect(err, gs.IsNil) if strings.Contains(line, "BOGUS") { continue } strVal := reverseMap[pack.Message.GetSeverity()] c.Expect(strings.Contains(line, strVal), gs.IsTrue) } }) }) }) }
func HttpInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'` c.Specify("A HttpInput", func() { httpInput := HttpInput{} ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) c.Specify("honors time ticker to flush", func() { startInput := func() { go func() { err := httpInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) ith.PackSupply <- ith.Pack // Spin up a http server server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876) c.Expect(err, gs.IsNil) go server.Start("/") time.Sleep(10 * time.Millisecond) config := httpInput.ConfigStruct().(*HttpInputConfig) decoderName := "TestDecoder" config.DecoderName = decoderName config.Url = "http://localhost:9876/" tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("short circuits packs into the router", func() { startInput := func() { go func() { err := httpInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) ith.PackSupply <- ith.Pack config := httpInput.ConfigStruct().(*HttpInputConfig) config.Url = "http://localhost:9876/" tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) err := httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("supports configuring HTTP Basic Authentication", func() { startInput := func() { go func() { err := httpInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) ith.PackSupply <- ith.Pack // Spin up a http server which expects username "user" and password "password" server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875) c.Expect(err, gs.IsNil) go server.Start("/BasicAuthTest") time.Sleep(10 * time.Millisecond) config := httpInput.ConfigStruct().(*HttpInputConfig) decoderName := "TestDecoder" config.DecoderName = decoderName config.Url = "http://localhost:9875/BasicAuthTest" config.User = "******" config.Password = "******" tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // we expect a statuscode 200 (i.e. success) pack := <-dRunnerInChan statusCode, ok := pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("supports configuring a different HTTP method", func() { startInput := func() { go func() { err := httpInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) ith.PackSupply <- ith.Pack // Spin up a http server which expects requests with method "POST" server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874) c.Expect(err, gs.IsNil) go server.Start("/PostTest") time.Sleep(10 * time.Millisecond) config := httpInput.ConfigStruct().(*HttpInputConfig) decoderName := "TestDecoder" config.DecoderName = decoderName config.Url = "http://localhost:9874/PostTest" config.Method = "POST" tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // we expect a statuscode 200 (i.e. success) pack := <-dRunnerInChan statusCode, ok := pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("supports configuring HTTP headers", func() { startInput := func() { go func() { err := httpInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) ith.PackSupply <- ith.Pack // Spin up a http server which expects requests with method "POST" server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873) c.Expect(err, gs.IsNil) go server.Start("/HeadersTest") time.Sleep(10 * time.Millisecond) config := httpInput.ConfigStruct().(*HttpInputConfig) decoderName := "TestDecoder" config.DecoderName = decoderName config.Url = "http://localhost:9873/HeadersTest" config.Headers = map[string]string{"Accept": "text/plain"} tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // we expect a statuscode 200 (i.e. success) pack := <-dRunnerInChan statusCode, ok := pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("supports configuring a request body", func() { startInput := func() { go func() { err := httpInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) ith.PackSupply <- ith.Pack // Spin up a http server that echoes back the request body server, err := plugins_ts.NewHttpBodyServer("localhost", 9872) c.Expect(err, gs.IsNil) go server.Start("/BodyTest") time.Sleep(10 * time.Millisecond) config := httpInput.ConfigStruct().(*HttpInputConfig) decoderName := "TestDecoder" config.DecoderName = decoderName config.Url = "http://localhost:9872/BodyTest" config.Method = "POST" config.Body = json_post tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() pack := <-dRunnerInChan c.Expect(*pack.Message.Payload, gs.Equals, json_post) // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()) httpInput.Stop() runtime.Gosched() // Yield so the stop can happen before we return. time.Sleep(50 * time.Millisecond) }) }
func MultiDecoderSpec(c gospec.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) // initializes Globals() c.Specify("A MultiDecoder", func() { subsTOML := `[StartsWithM] type = "PayloadRegexDecoder" match_regex = '^(?P<TheData>m.*)' log_errors = true [StartsWithM.message_fields] StartsWithM = "%TheData%" [StartsWithS] type = "PayloadRegexDecoder" match_regex = '^(?P<TheData>s.*)' log_errors = true [StartsWithS.message_fields] StartsWithS = "%TheData%" [StartsWithM2] type = "PayloadRegexDecoder" match_regex = '^(?P<TheData>m.*)' log_errors = true [StartsWithM2.message_fields] StartsWithM2 = "%TheData%" ` RegisterPlugin("PayloadRegexDecoder", func() interface{} { return &PayloadRegexDecoder{} }) defer delete(AvailablePlugins, "PayloadRegexDecoder") var configFile ConfigFile _, err := toml.Decode(subsTOML, &configFile) c.Assume(err, gs.IsNil) decoder := new(MultiDecoder) decoder.SetName("MyMultiDecoder") decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*MultiDecoderConfig) supply := make(chan *PipelinePack, 1) pack := NewPipelinePack(supply) mSection, ok := configFile["StartsWithM"] c.Assume(ok, gs.IsTrue) mMaker, err := NewPluginMaker("StartsWithM", pConfig, mSection) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["StartsWithM"] = mMaker conf.Subs = []string{"StartsWithM"} errMsg := "All subdecoders failed." dRunner := pipelinemock.NewMockDecoderRunner(ctrl) // An error will be spit out b/c there's no real *dRunner in there; // doesn't impact the tests. dRunner.EXPECT().LogError(gomock.Any()) c.Specify("decodes simple messages", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "matching text" pack.Message.SetPayload(regex_data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) value, ok := pack.Message.GetFieldValue("StartsWithM") c.Assume(ok, gs.IsTrue) c.Expect(value, gs.Equals, regex_data) }) c.Specify("returns an error if all decoders fail", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "non-matching text" pack.Message.SetPayload(regex_data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) }) c.Specify("logs subdecoder failures when configured to do so", func() { conf.LogSubErrors = true err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "non-matching text" pack.Message.SetPayload(regex_data) // Expect that we log an error for undecoded message. dRunner.EXPECT().LogError(fmt.Errorf( "Subdecoder 'StartsWithM' decode error: No match: %s", regex_data)).AnyTimes() packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) }) c.Specify("sets subdecoder runner correctly", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) // Call LogError to appease the angry gomock gods. dRunner.LogError(errors.New("foo")) // Now create a real *dRunner, pass it in, make sure a wrapper // gets handed to the subdecoder. dr := NewDecoderRunner(decoder.Name, decoder, 10) decoder.SetDecoderRunner(dr) sub := decoder.Decoders[0] subRunner := sub.(*PayloadRegexDecoder).dRunner c.Expect(subRunner.Name(), gs.Equals, fmt.Sprintf("%s-StartsWithM", decoder.Name)) c.Expect(subRunner.Decoder(), gs.Equals, sub) }) c.Specify("with multiple registered decoders", func() { sSection, ok := configFile["StartsWithS"] c.Assume(ok, gs.IsTrue) sMaker, err := NewPluginMaker("StartsWithS", pConfig, sSection) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["StartsWithS"] = sMaker m2Section, ok := configFile["StartsWithM2"] c.Assume(ok, gs.IsTrue) m2Maker, err := NewPluginMaker("StartsWithM2", pConfig, m2Section) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["StartsWithM2"] = m2Maker conf.Subs = append(conf.Subs, "StartsWithS", "StartsWithM2") // Two more subdecoders means two more LogError calls. dRunner.EXPECT().LogError(gomock.Any()).Times(2) c.Specify("defaults to `first-wins` cascading", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("on a first match condition", func() { pack.Message.SetPayload("match first") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("and a second match condition", func() { pack.Message.SetPayload("second match") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("returning an error if they all fail", func() { pack.Message.SetPayload("won't match") packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) }) c.Specify("and using `all` cascading", func() { conf.CascadeStrategy = "all" err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("matches multiples when appropriate", func() { pack.Message.SetPayload("matches twice") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsTrue) }) c.Specify("matches singles when appropriate", func() { pack.Message.SetPayload("second match") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("returns an error if they all fail", func() { pack.Message.SetPayload("won't match") packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) }) }) }) c.Specify("A MultiDecoder w/ MultiOutput", func() { subsTOML := `[sub0] type = "MultiOutputDecoder" [sub1] type = "MultiOutputDecoder" [sub2] type = "MultiOutputDecoder" ` RegisterPlugin("MultiOutputDecoder", func() interface{} { return &MultiOutputDecoder{} }) defer delete(AvailablePlugins, "MultiOutputDecoder") var configFile ConfigFile _, err := toml.Decode(subsTOML, &configFile) decoder := new(MultiDecoder) decoder.SetName("MyMultiDecoder") decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*MultiDecoderConfig) conf.CascadeStrategy = "all" supply := make(chan *PipelinePack, 10) pack := NewPipelinePack(supply) sub0Section, ok := configFile["sub0"] c.Assume(ok, gs.IsTrue) sub0Maker, err := NewPluginMaker("sub0", pConfig, sub0Section) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["sub0"] = sub0Maker sub1Section, ok := configFile["sub1"] c.Assume(ok, gs.IsTrue) sub1Maker, err := NewPluginMaker("sub1", pConfig, sub1Section) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["sub1"] = sub1Maker sub2Section, ok := configFile["sub2"] c.Assume(ok, gs.IsTrue) sub2Maker, err := NewPluginMaker("sub2", pConfig, sub2Section) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["sub2"] = sub2Maker conf.Subs = []string{"sub0", "sub1", "sub2"} dRunner := pipelinemock.NewMockDecoderRunner(ctrl) c.Specify("always tries to decode all packs", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 15) }) }) }
func DecoderSpec(c gs.Context) { t := new(ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := pipeline.NewPipelineConfig(nil) c.Specify("A SandboxDecoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) c.Specify("that uses lpeg and inject_message", func() { dRunner.EXPECT().Name().Return("serialize") conf.ScriptFilename = "../lua/testsupport/decoder.lua" err := decoder.Init(conf) c.Assume(err, gs.IsNil) c.Specify("decodes simple messages", func() { data := "1376389920 debug id=2321 url=example.com item=1" decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1376389920000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("id") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "2321") value, ok = pack.Message.GetFieldValue("url") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "example.com") value, ok = pack.Message.GetFieldValue("item") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "1") decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "1376389920 bogus id=2321 url=example.com item=1" decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) c.Specify("Preserves data", func() { conf.ScriptFilename = "../lua/testsupport/serialize.lua" conf.PreserveData = true err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) decoder.Shutdown() _, err = os.Stat("sandbox_preservation/serialize.data") c.Expect(err, gs.IsNil) err = os.Remove("sandbox_preservation/serialize.data") c.Expect(err, gs.IsNil) }) }) c.Specify("that only uses write_message", func() { conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua" dRunner.EXPECT().Name().Return("write_message") err := decoder.Init(conf) decoder.SetDecoderRunner(dRunner) c.Assume(err, gs.IsNil) c.Specify("adds a string field to the message", func() { data := "string field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(string), gs.Equals, "foo") }) c.Specify("adds a numeric field to the message", func() { data := "num field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(float64), gs.Equals, float64(1)) }) c.Specify("adds a boolean field to the message", func() { data := "bool field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(bool), gs.Equals, true) }) c.Specify("sets type and payload", func() { data := "set type and payload" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) c.Expect(pack.Message.GetType(), gs.Equals, "my_type") c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload") }) c.Specify("sets field value with representation", func() { data := "set field value with representation" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("rep") c.Expect(len(fields), gs.Equals, 1) field := fields[0] values := field.GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "foo") c.Expect(field.GetRepresentation(), gs.Equals, "representation") }) c.Specify("sets multiple field string values", func() { data := "set multiple field string values" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("multi") c.Expect(len(fields), gs.Equals, 2) values := fields[0].GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "first") values = fields[1].GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "second") }) c.Specify("sets field string array value", func() { data := "set field string array value" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("array") c.Expect(len(fields), gs.Equals, 1) values := fields[0].GetValueString() c.Expect(len(values), gs.Equals, 2) c.Expect(values[0], gs.Equals, "first") c.Expect(values[1], gs.Equals, "second") }) }) }) c.Specify("A Multipack SandboxDecoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua" supply := make(chan *pipeline.PipelinePack, 3) pack := pipeline.NewPipelinePack(supply) pack.Message = getTestMessage() pack1 := pipeline.NewPipelinePack(supply) pack2 := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") c.Specify("decodes into multiple packs", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) gomock.InOrder( dRunner.EXPECT().NewPack().Return(pack1), dRunner.EXPECT().NewPack().Return(pack2), ) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 3) c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one") c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two") c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three") for i := 0; i < 1; i++ { c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST") c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name") c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec") c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6)) } decoder.Shutdown() }) }) c.Specify("Linux Cpu Stats decoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/linux_loadavg.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes a message", func() { payload := "0.00 0.01 0.05 3/153 660\n" pack.Message.SetPayload(payload) f, err := message.NewField("FilePath", "/proc/loadavg", "") c.Assume(err, gs.IsNil) pack.Message.AddField(f) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("1MinAvg") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, 0.00) value, ok = pack.Message.GetFieldValue("5MinAvg") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, 0.01) value, ok = pack.Message.GetFieldValue("15MinAvg") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, 0.05) value, ok = pack.Message.GetFieldValue("NumProcesses") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(3)) value, ok = pack.Message.GetFieldValue("FilePath") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, "/proc/loadavg") }) c.Specify("decodes an invalid message", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("Linux Mem Stats decoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/linux_memstats.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes a message", func() { payload := `MemTotal: 4047616 kB MemFree: 3135780 kB HugePages_Free: 0 ` pack.Message.SetPayload(payload) f, err := message.NewField("FilePath", "/proc/meminfo", "") c.Assume(err, gs.IsNil) pack.Message.AddField(f) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("MemTotal") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, 4.047616e+06) value, ok = pack.Message.GetFieldValue("MemFree") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, 3.13578e+06) value, ok = pack.Message.GetFieldValue("HugePages_Free") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(0)) value, ok = pack.Message.GetFieldValue("FilePath") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, "/proc/meminfo") }) c.Specify("decodes an invalid message", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("Linux Disk Stats decoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/linux_diskstats.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes a message", func() { payload := " 13903 11393 969224 49444 10780 10161 1511920 4104 0 5064 53468\n" pack.Message.SetPayload(payload) f, err := message.NewField("FilePath", "/sys/block/sda/stat", "") c.Assume(err, gs.IsNil) pack.Message.AddField(f) f, err = message.NewField("TickerInterval", int64(2), "") c.Assume(err, gs.IsNil) pack.Message.AddField(f) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} // These are in the same order the payload should be value, ok = pack.Message.GetFieldValue("ReadsCompleted") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(13903)) value, ok = pack.Message.GetFieldValue("ReadsMerged") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(11393)) value, ok = pack.Message.GetFieldValue("SectorsRead") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(969224)) value, ok = pack.Message.GetFieldValue("TimeReading") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(49444)) value, ok = pack.Message.GetFieldValue("WritesCompleted") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(10780)) value, ok = pack.Message.GetFieldValue("WritesMerged") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(10161)) value, ok = pack.Message.GetFieldValue("SectorsWritten") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(1511920)) value, ok = pack.Message.GetFieldValue("TimeWriting") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(4104)) value, ok = pack.Message.GetFieldValue("NumIOInProgress") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(0)) value, ok = pack.Message.GetFieldValue("TimeDoingIO") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(5064)) value, ok = pack.Message.GetFieldValue("WeightedTimeDoingIO") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(53468)) value, ok = pack.Message.GetFieldValue("TickerInterval") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(2)) value, ok = pack.Message.GetFieldValue("FilePath") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, "/sys/block/sda/stat") }) c.Specify("decodes a message with no leading space", func() { payload := "19092852 0 510563170 15817012 46452019 0 1546950712 262535124 0 23823976 278362684\n" pack.Message.SetPayload(payload) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) value, ok := pack.Message.GetFieldValue("ReadsCompleted") c.Expect(ok, gs.IsTrue) c.Expect(value, gs.Equals, float64(19092852)) }) c.Specify("decodes an invalid message", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("Nginx access log decoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/nginx_access.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\"" conf.Config["user_agent_transform"] = true supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes simple messages", func() { data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\"" pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1392050801000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("remote_addr") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "127.0.0.1") value, ok = pack.Message.GetFieldValue("user_agent_browser") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Firefox") value, ok = pack.Message.GetFieldValue("user_agent_version") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(26)) value, ok = pack.Message.GetFieldValue("user_agent_os") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Linux") _, ok = pack.Message.GetFieldValue("http_user_agent") c.Expect(ok, gs.Equals, false) value, ok = pack.Message.GetFieldValue("body_bytes_sent") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(0)) value, ok = pack.Message.GetFieldValue("status") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(304)) decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("Apache access log decoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/apache_access.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" conf.Config["user_agent_transform"] = true supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes simple messages", func() { data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\"" pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1392050801000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("remote_addr") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "127.0.0.1") value, ok = pack.Message.GetFieldValue("user_agent_browser") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Firefox") value, ok = pack.Message.GetFieldValue("user_agent_version") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(26)) value, ok = pack.Message.GetFieldValue("user_agent_os") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Linux") _, ok = pack.Message.GetFieldValue("http_user_agent") c.Expect(ok, gs.Equals, false) value, ok = pack.Message.GetFieldValue("body_bytes_sent") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(0)) value, ok = pack.Message.GetFieldValue("status") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(304)) decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("rsyslog decoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/rsyslog.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["type"] = "MyTestFormat" conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" conf.Config["tz"] = "America/Los_Angeles" supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes simple messages", func() { data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n" pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) // Syslog timestamp doesn't support year, so we have to calculate // it for the current year or else this test will fail every // January. year := time.Now().Year() tStr := fmt.Sprintf("%d Feb 10 12:58:58 -0800", year) t, err := time.Parse("2006 Jan 02 15:04:05 -0700", tStr) c.Assume(err, gs.IsNil) unixT := t.UnixNano() c.Expect(pack.Message.GetTimestamp(), gs.Equals, unixT) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4)) c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost") c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322)) c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.") c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"]) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("programname") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "widget") value, ok = pack.Message.GetFieldValue("syslogfacility") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(3)) value, ok = pack.Message.GetFieldValue("timegenerated") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(1392065939000000000)) decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("mysql decoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["truncate_sql"] = int64(5) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decode standard slow query log", func() { data := `# User@Host: syncrw[syncrw] @ [127.0.0.1] # Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 use widget; SET last_insert_id=999,insert_id=1000,timestamp=1399500744; # administrator command: do something /* [queryName=FIND_ITEMS] */ SELECT * FROM widget WHERE id = 10;` pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1399500744000000000)) c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...") c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query") decoder.Shutdown() }) }) c.Specify("mariadb decoder", func() { decoder := new(SandboxDecoder) decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.Config = make(map[string]interface{}) conf.Config["truncate_sql"] = int64(5) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decode standard slow query log", func() { data := `# User@Host: syncrw[syncrw] @ [127.0.0.1] # Thread_id: 110804 Schema: weave0 QC_hit: No # Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198 SET timestamp=1399500744; /* [queryName=FIND_ITEMS] */ SELECT * FROM widget WHERE id = 10;` pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1399500744000000000)) c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...") c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query") decoder.Shutdown() }) }) }
func ProcessDirectoryInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.PackSupply <- ith.Pack //ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes() ith.DecodeChan = make(chan *PipelinePack) //mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner) //mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).AnyTimes() c.Specify("A ProcessDirectoryInput", func() { pdiInput := ProcessDirectoryInput{} //ith.MockInputRunner.EXPECT().Name().Return("logger").AnyTimes() // Ignore all messages //ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes() config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig) workingDir, err := os.Getwd() c.Assume(err, gs.IsNil) config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes") pConfig := NewPipelineConfig(nil) // `Ticker` is the last thing called during the setup part of the // input's `Run` method, so it triggers a waitgroup that tests can // wait on when they need to ensure initialization has finished. var started sync.WaitGroup started.Add(1) tickChan := make(chan time.Time, 1) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do( func() { started.Done() }) // Similarly we use a waitgroup to signal when LogMessage has been // called to know when reloads have completed. Warning: If you call // expectLogMessage with a msg that is never passed to LogMessage and // then you call loaded.Wait() then your test will hang and never // complete. var loaded sync.WaitGroup expectLogMessage := func(msg string) { loaded.Add(1) ith.MockInputRunner.EXPECT().LogMessage(msg).Do( func(msg string) { loaded.Done() }) } // Same name => same content. paths := []string{ filepath.Join(config.ProcessDir, "100", "h0.toml"), filepath.Join(config.ProcessDir, "100", "h1.toml"), filepath.Join(config.ProcessDir, "200", "h0.toml"), filepath.Join(config.ProcessDir, "300", "h1.toml"), } copyFile := func(src, dest string) { inFile, err := os.Open(src) c.Assume(err, gs.IsNil) outFile, err := os.Create(dest) c.Assume(err, gs.IsNil) _, err = io.Copy(outFile, inFile) c.Assume(err, gs.IsNil) } err = pdiInput.Init(config) c.Expect(err, gs.IsNil) ith.MockHelper.EXPECT().PipelineConfig().Times(4).Return(pConfig) for _, p := range paths { expectLogMessage("Added: " + p) } go pdiInput.Run(ith.MockInputRunner, ith.MockHelper) defer func() { pdiInput.Stop() for _, entry := range pdiInput.inputs { entry.ir.Input().Stop() } }() started.Wait() c.Specify("loads scheduled jobs", func() { pathIndex := func(name string) (i int) { var p string for i, p = range paths { if name == p { return } } return -1 } for name, entry := range pdiInput.inputs { i := pathIndex(name) // Make sure each file path got registered. c.Expect(i, gs.Not(gs.Equals), -1) dirName := filepath.Base(filepath.Dir(name)) dirInt, err := strconv.Atoi(dirName) c.Expect(err, gs.IsNil) // And that the ticker interval was read correctly. c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval) } }) c.Specify("discovers and adds a new job", func() { // Copy one of the files to register a new process. newPath := filepath.Join(config.ProcessDir, "300", "h0.toml") copyFile(paths[0], newPath) defer func() { err := os.Remove(newPath) c.Assume(err, gs.IsNil) }() // Set up expectations and trigger process dir reload. ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) expectLogMessage("Added: " + newPath) tickChan <- time.Now() loaded.Wait() // Make sure our plugin was loaded. c.Expect(len(pdiInput.inputs), gs.Equals, 5) newEntry, ok := pdiInput.inputs[newPath] c.Expect(ok, gs.IsTrue) c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300)) }) c.Specify("removes a deleted job", func() { err := os.Remove(paths[3]) c.Assume(err, gs.IsNil) defer func() { copyFile(paths[1], paths[3]) }() // Set up expectations and trigger process dir reload. ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) expectLogMessage("Removed: " + paths[3]) tickChan <- time.Now() loaded.Wait() // Make sure our plugin was deleted. c.Expect(len(pdiInput.inputs), gs.Equals, 3) }) c.Specify("notices a changed job", func() { // Overwrite one job w/ a slightly different one. copyFile(paths[0], paths[3]) defer copyFile(paths[1], paths[3]) // Set up expectations and trigger process dir reload. ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig).AnyTimes() expectLogMessage("Removed: " + paths[3]) expectLogMessage("Added: " + paths[3]) tickChan <- time.Now() loaded.Wait() // Make sure the new config was loaded. c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals, "hello world\n") }) }) }
func DecoderSpec(c gs.Context) { t := new(ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() // NewPipelineConfig sets up Globals which is needed for the // pipeline.Prepend*Dir functions to not die during plugin Init(). _ = pipeline.NewPipelineConfig(nil) c.Specify("A SandboxDecoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) conf.ScriptType = "lua" c.Specify("that uses lpeg and inject_message", func() { dRunner.EXPECT().Name().Return("serialize") conf.ScriptFilename = "../lua/testsupport/decoder.lua" err := decoder.Init(conf) c.Assume(err, gs.IsNil) c.Specify("decodes simple messages", func() { data := "1376389920 debug id=2321 url=example.com item=1" decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1376389920000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("id") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "2321") value, ok = pack.Message.GetFieldValue("url") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "example.com") value, ok = pack.Message.GetFieldValue("item") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "1") decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "1376389920 bogus id=2321 url=example.com item=1" decoder.SetDecoderRunner(dRunner) pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) c.Specify("Preserves data", func() { conf.ScriptFilename = "../lua/testsupport/serialize.lua" conf.PreserveData = true err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) decoder.Shutdown() _, err = os.Stat("sandbox_preservation/serialize.data") c.Expect(err, gs.IsNil) err = os.Remove("sandbox_preservation/serialize.data") c.Expect(err, gs.IsNil) }) }) c.Specify("that only uses write_message", func() { conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua" dRunner.EXPECT().Name().Return("write_message") err := decoder.Init(conf) decoder.SetDecoderRunner(dRunner) c.Assume(err, gs.IsNil) c.Specify("adds a string field to the message", func() { data := "string field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(string), gs.Equals, "foo") }) c.Specify("adds a numeric field to the message", func() { data := "num field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(float64), gs.Equals, float64(1)) }) c.Specify("adds a boolean field to the message", func() { data := "bool field scribble" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) value, ok := pack.Message.GetFieldValue("scribble") c.Expect(ok, gs.IsTrue) c.Expect(value.(bool), gs.Equals, true) }) c.Specify("sets type and payload", func() { data := "set type and payload" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) c.Expect(pack.Message.GetType(), gs.Equals, "my_type") c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload") }) c.Specify("sets field value with representation", func() { data := "set field value with representation" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("rep") c.Expect(len(fields), gs.Equals, 1) field := fields[0] values := field.GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "foo") c.Expect(field.GetRepresentation(), gs.Equals, "representation") }) c.Specify("sets multiple field string values", func() { data := "set multiple field string values" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("multi") c.Expect(len(fields), gs.Equals, 2) values := fields[0].GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "first") values = fields[1].GetValueString() c.Expect(len(values), gs.Equals, 1) c.Expect(values[0], gs.Equals, "second") }) c.Specify("sets field string array value", func() { data := "set field string array value" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 1) c.Expect(packs[0], gs.Equals, pack) fields := pack.Message.FindAllFields("array") c.Expect(len(fields), gs.Equals, 1) values := fields[0].GetValueString() c.Expect(len(values), gs.Equals, 2) c.Expect(values[0], gs.Equals, "first") c.Expect(values[1], gs.Equals, "second") }) }) }) c.Specify("A Multipack SandboxDecoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua" conf.ScriptType = "lua" supply := make(chan *pipeline.PipelinePack, 3) pack := pipeline.NewPipelinePack(supply) pack.Message = getTestMessage() pack1 := pipeline.NewPipelinePack(supply) pack2 := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") c.Specify("decodes into multiple packs", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) gomock.InOrder( dRunner.EXPECT().NewPack().Return(pack1), dRunner.EXPECT().NewPack().Return(pack2), ) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 3) c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one") c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two") c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three") for i := 0; i < 1; i++ { c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST") c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name") c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec") c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6)) } decoder.Shutdown() }) }) c.Specify("Nginx access log decoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/nginx_access.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.ScriptType = "lua" conf.Config = make(map[string]interface{}) conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\"" conf.Config["user_agent_transform"] = true supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes simple messages", func() { data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\"" pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1392050801000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7)) var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("remote_addr") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "127.0.0.1") value, ok = pack.Message.GetFieldValue("user_agent_browser") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Firefox") value, ok = pack.Message.GetFieldValue("user_agent_version") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(26)) value, ok = pack.Message.GetFieldValue("user_agent_os") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "Linux") _, ok = pack.Message.GetFieldValue("http_user_agent") c.Expect(ok, gs.Equals, false) value, ok = pack.Message.GetFieldValue("body_bytes_sent") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(0)) value, ok = pack.Message.GetFieldValue("status") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(304)) decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) c.Specify("rsyslog decoder", func() { decoder := new(SandboxDecoder) conf := decoder.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/decoders/rsyslog.lua" conf.ModuleDirectory = "../../../../../../modules" conf.MemoryLimit = 8e6 conf.ScriptType = "lua" conf.Config = make(map[string]interface{}) conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" conf.Config["tz"] = "America/Los_Angeles" supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) dRunner := pm.NewMockDecoderRunner(ctrl) dRunner.EXPECT().Name().Return("SandboxDecoder") err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("decodes simple messages", func() { data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost kernel: imklog 5.8.6, log source = /proc/kmsg started.\n" pack.Message.SetPayload(data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1392065938000000000)) c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4)) c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost") c.Expect(pack.Message.GetPayload(), gs.Equals, "imklog 5.8.6, log source = /proc/kmsg started.") var ok bool var value interface{} value, ok = pack.Message.GetFieldValue("syslogtag") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, "kernel:") value, ok = pack.Message.GetFieldValue("syslogfacility") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(3)) value, ok = pack.Message.GetFieldValue("timegenerated") c.Expect(ok, gs.Equals, true) c.Expect(value, gs.Equals, float64(1392065939000000000)) decoder.Shutdown() }) c.Specify("decodes an invalid messages", func() { data := "bogus message" pack.Message.SetPayload(data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data) c.Expect(decoder.processMessageFailures, gs.Equals, int64(1)) decoder.Shutdown() }) }) }
func FilePollingInputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) tmpFileName := fmt.Sprintf("filepollinginput-test-%d", time.Now().UnixNano()) tmpFilePath := filepath.Join(os.TempDir(), tmpFileName) defer func() { ctrl.Finish() os.Remove(tmpFilePath) }() pConfig := NewPipelineConfig(nil) var wg sync.WaitGroup errChan := make(chan error, 1) tickChan := make(chan time.Time) retPackChan := make(chan *PipelinePack, 2) defer close(retPackChan) c.Specify("A FilePollingInput", func() { input := new(FilePollingInput) ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 2) ith.PackSupply <- ith.Pack ith.PackSupply <- ith.Pack config := input.ConfigStruct().(*FilePollingInputConfig) config.FilePath = tmpFilePath c.Specify("That is started", func() { startInput := func() { wg.Add(1) go func() { errChan <- input.Run(ith.MockInputRunner, ith.MockHelper) wg.Done() }() } inputName := "FilePollingInput" ith.MockInputRunner.EXPECT().Name().Return(inputName).AnyTimes() ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) c.Specify("gets updated information when reading a file", func() { c.Specify("injects messages into the pipeline when not configured with a decoder", func() { ith.MockInputRunner.EXPECT().Inject(ith.Pack).Do(func(pack *PipelinePack) { retPackChan <- pack }).Times(2) err := input.Init(config) c.Assume(err, gs.IsNil) c.Expect(input.DecoderName, gs.Equals, "") }) c.Specify("sends messages to a decoder when configured with a decoder", func() { decoderName := "ScribbleDecoder" config.DecoderName = decoderName mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) mockDecoderRunner.EXPECT().InChan().Return(retPackChan) ith.MockHelper.EXPECT().DecoderRunner(decoderName, fmt.Sprintf("%s-%s", inputName, decoderName)).Return(mockDecoderRunner, true) err := input.Init(config) c.Assume(err, gs.IsNil) c.Expect(input.DecoderName, gs.Equals, decoderName) }) startInput() f, err := os.Create(tmpFilePath) c.Expect(err, gs.IsNil) _, err = f.Write([]byte("test1")) c.Expect(err, gs.IsNil) c.Expect(f.Close(), gs.IsNil) tickChan <- time.Now() pack := <-retPackChan c.Expect(pack.Message.GetPayload(), gs.Equals, "test1") f, err = os.Create(tmpFilePath) c.Expect(err, gs.IsNil) _, err = f.Write([]byte("test2")) c.Expect(err, gs.IsNil) c.Expect(f.Close(), gs.IsNil) tickChan <- time.Now() pack = <-retPackChan c.Expect(pack.Message.GetPayload(), gs.Equals, "test2") input.Stop() wg.Wait() c.Expect(<-errChan, gs.IsNil) }) }) }) }