func DashboardOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := pipeline.NewPipelineConfig(nil) dashboardOutput := new(DashboardOutput) dashboardOutput.pConfig = pConfig oth := plugins_ts.NewOutputTestHelper(ctrl) oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl) errChan := make(chan error, 1) startOutput := func() { go func() { errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper) }() } if runtime.GOOS != "windows" { c.Specify("A DashboardOutput", func() { tmpdir, err := ioutil.TempDir("", "dashboard_output_test") c.Assume(err, gs.IsNil) config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig) config.WorkingDirectory = tmpdir c.Specify("Init halts if basedirectory is not writable", func() { err := os.MkdirAll(tmpdir, 0400) c.Assume(err, gs.IsNil) defer os.RemoveAll(tmpdir) err = dashboardOutput.Init(config) c.Assume(err, gs.Not(gs.IsNil)) }) c.Specify("that is running", func() { startedChan := make(chan bool, 1) defer close(startedChan) ts := httptest.NewUnstartedServer(nil) dashboardOutput.starterFunc = func(hli *DashboardOutput) error { ts.Start() startedChan <- true return nil } ticker := make(chan time.Time) inChan := make(chan *pipeline.PipelinePack, 1) recycleChan := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(recycleChan) pack.Message = pipeline_ts.GetTestMessage() oth.MockOutputRunner.EXPECT().InChan().Return(inChan) oth.MockOutputRunner.EXPECT().Ticker().Return(ticker) err := os.MkdirAll(tmpdir, 0700) c.Assume(err, gs.IsNil) defer os.RemoveAll(tmpdir) dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Noop }) c.Specify("sets custom http headers", func() { config.Headers = http.Header{ "One": []string{"two", "three"}, "Four": []string{"five", "six", "seven"}, } err = dashboardOutput.Init(config) c.Assume(err, gs.IsNil) ts.Config = dashboardOutput.server startOutput() inChan <- pack <-startedChan resp, err := http.Get(ts.URL) c.Assume(err, gs.IsNil) resp.Body.Close() c.Assume(resp.StatusCode, gs.Equals, 200) // Verify headers are there eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"]) c.Expect(eq, gs.IsTrue) eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"]) c.Expect(eq, gs.IsTrue) }) close(inChan) c.Expect(<-errChan, gs.IsNil) ts.Close() }) }) } }
func UdpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) udpOutput := new(UdpOutput) config := udpOutput.ConfigStruct().(*UdpOutputConfig) oth := plugins_ts.NewOutputTestHelper(ctrl) encoder := new(plugins.PayloadEncoder) encoder.Init(new(plugins.PayloadEncoderConfig)) inChan := make(chan *pipeline.PipelinePack, 1) rChan := make(chan *pipeline.PipelinePack, 1) var wg sync.WaitGroup var rAddr net.Addr c.Specify("A UdpOutput", func() { msg := pipeline_ts.GetTestMessage() payload := "Write me out to the network." msg.SetPayload(payload) pack := pipeline.NewPipelinePack(rChan) pack.Message = msg oth.MockOutputRunner.EXPECT().InChan().Return(inChan) oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes() oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) c.Specify("using UDP", func() { addr := "127.0.0.1:45678" config.Address = addr ch := make(chan string, 1) collectData := func() { conn, err := net.ListenPacket("udp", addr) if err != nil { ch <- err.Error() return } ch <- "ready" b := make([]byte, 1000) var n int n, rAddr, _ = conn.ReadFrom(b) ch <- string(b[:n]) conn.Close() } go collectData() result := <-ch // Wait for server to be ready. c.Assume(result, gs.Equals, "ready") c.Specify("writes out to the network", func() { err := udpOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() inChan <- pack result = <-ch c.Expect(result, gs.Equals, payload) close(inChan) wg.Wait() }) c.Specify("uses the specified local address", func() { config.LocalAddress = "localhost:12345" err := udpOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() inChan <- pack result = <-ch c.Expect(result, gs.Equals, payload) c.Expect(rAddr.Network(), gs.Equals, "udp") c.Expect(rAddr.String(), gs.Equals, "127.0.0.1:12345") close(inChan) wg.Wait() }) }) c.Specify("using Unix datagrams", func() { if runtime.GOOS == "windows" { return } testUnixAddr := func() string { f, err := ioutil.TempFile("", "_heka_test_sock") c.Assume(err, gs.IsNil) addr := f.Name() f.Close() os.Remove(addr) return addr } config.Address = testUnixAddr() config.Net = "unixgram" ch := make(chan string, 1) var wg sync.WaitGroup var rAddr net.Addr collectData := func() { conn, err := net.ListenPacket("unixgram", config.Address) if err != nil { ch <- err.Error() return } ch <- "ready" b := make([]byte, 1000) var n int n, rAddr, _ = conn.ReadFrom(b) ch <- string(b[:n]) conn.Close() err = os.Remove(config.Address) var errMsg string if err != nil { errMsg = err.Error() } ch <- errMsg } go collectData() result := <-ch // Wait for server to be ready. c.Assume(result, gs.Equals, "ready") c.Specify("writes out to the network", func() { err := udpOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() inChan <- pack result = <-ch c.Expect(result, gs.Equals, payload) close(inChan) wg.Wait() result = <-ch c.Expect(result, gs.Equals, "") }) c.Specify("uses the specified local address", func() { config.LocalAddress = testUnixAddr() err := udpOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() inChan <- pack result = <-ch c.Expect(result, gs.Equals, payload) c.Expect(rAddr.Network(), gs.Equals, "unixgram") c.Expect(rAddr.String(), gs.Equals, config.LocalAddress) close(inChan) wg.Wait() }) }) }) c.Specify("drop message contents if their size is bigger than allowed UDP datagram size", func() { huge_msg := pipeline_ts.GetTestMessage() payload := strings.Repeat("2", 131014) huge_msg.SetPayload(payload) huge_pack := pipeline.NewPipelinePack(rChan) huge_pack.Message = huge_msg oth.MockOutputRunner.EXPECT().InChan().Return(inChan) oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes() oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(huge_pack).Return(encoder.Encode(huge_pack)) oth.MockOutputRunner.EXPECT().LogError(fmt.Errorf("Message has exceeded allowed UDP data size: 131014 > 65507")) config.Address = "localhost:12345" err := udpOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() inChan <- huge_pack close(inChan) wg.Wait() }) c.Specify("checks validation of of Maximum message size limit", func() { config.Address = "localhost:12345" config.MaxMessageSize = 100 err := udpOutput.Init(config) c.Assume(err.Error(), gs.Equals, "Maximum message size can't be smaller than 512 bytes.") }) }
func SmtpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() oth := plugins_ts.NewOutputTestHelper(ctrl) var wg sync.WaitGroup inChan := make(chan *PipelinePack, 1) pConfig := NewPipelineConfig(nil) encoder := new(plugins.PayloadEncoder) econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig) econfig.AppendNewlines = false encoder.Init(econfig) c.Specify("A SmtpOutput", func() { smtpOutput := new(SmtpOutput) config := smtpOutput.ConfigStruct().(*SmtpOutputConfig) config.SendTo = []string{"root"} msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() inChanCall.Return(inChan) runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes() runnerName.Return("SmtpOutput") oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes() c.Specify("send email payload message", func() { err := smtpOutput.Init(config) c.Assume(err, gs.IsNil) smtpOutput.sendFunction = testSendMail outStr := "Write me out to the network" pack.Message.SetPayload(outStr) go func() { wg.Add(1) smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper) wg.Done() }() inChan <- pack close(inChan) wg.Wait() }) }) // Use this test with a real server // c.Specify("Real SmtpOutput output", func() { // smtpOutput := new(SmtpOutput) // // config := smtpOutput.ConfigStruct().(*SmtpOutputConfig) // config.SendTo = []string{"root"} // // msg := pipeline_ts.GetTestMessage() // pack := NewPipelinePack(pConfig.InputRecycleChan()) // pack.Message = msg // pack.Decoded = true // inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() // inChanCall.Return(inChan) // runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes() // runnerName.Return("SmtpOutput") // oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes() // // c.Specify("send a real email essage", func() { // // err := smtpOutput.Init(config) // c.Assume(err, gs.IsNil) // // outStr := "Write me out to the network" // pack.Message.SetPayload(outStr) // go func() { // wg.Add(1) // smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper) // wg.Done() // }() // inChan <- pack // time.Sleep(1000) // allow time for the message output // close(inChan) // wg.Wait() // // manually check the mail // }) // }) }
func TcpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests") defer func() { ctrl.Finish() tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() pConfig := NewPipelineConfig(nil) c.Specify("TcpOutput", func() { origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir }() tcpOutput := new(TcpOutput) tcpOutput.SetName("test") config := tcpOutput.ConfigStruct().(*TcpOutputConfig) tcpOutput.Init(config) tickChan := make(chan time.Time) oth := plugins_ts.NewOutputTestHelper(ctrl) oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan) encoder := new(ProtobufEncoder) encoder.Init(nil) var wg sync.WaitGroup inChan := make(chan *PipelinePack, 1) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true outStr := "Write me out to the network" newpack := NewPipelinePack(nil) newpack.Message = msg newpack.Decoded = true newpack.Message.SetPayload(outStr) matchBytes, err := proto.Marshal(newpack.Message) c.Expect(err, gs.IsNil) inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() inChanCall.Return(inChan) c.Specify("doesn't use framing w/o ProtobufEncoder", func() { encoder := new(plugins.PayloadEncoder) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() close(inChan) wg.Wait() // We should fail if SetUseFraming is called since we didn't // EXPECT it. }) c.Specify("doesn't use framing if config says not to", func() { useFraming := false config.UseFraming = &useFraming err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() close(inChan) wg.Wait() // We should fail if SetUseFraming is called since we didn't // EXPECT it. }) c.Specify("writes out to the network", func() { collectData := func(ch chan string) { ln, err := net.Listen("tcp", "localhost:9125") if err != nil { ch <- err.Error() return } ch <- "ready" conn, err := ln.Accept() if err != nil { ch <- err.Error() return } b := make([]byte, 1000) n, _ := conn.Read(b) ch <- string(b[0:n]) conn.Close() ln.Close() } ch := make(chan string, 1) // don't block on put go collectData(ch) result := <-ch // wait for server err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().SetUseFraming(true) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes() pack.Message.SetPayload(outStr) wg.Add(1) go func() { err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(0)) inChan <- pack result = <-ch msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(1)) c.Expect(result, gs.Equals, string(matchBytes)) close(inChan) wg.Wait() // wait for output to finish shutting down }) c.Specify("far end not initially listening", func() { oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) pack.Message.SetPayload(outStr) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().SetUseFraming(true) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes() wg.Add(1) go func() { err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(0)) inChan <- pack for x := 0; x < 5 && msgcount == 0; x++ { msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount) time.Sleep(time.Duration(100) * time.Millisecond) } // After the message is queued start the collector. However, we // don't have a way guarantee a send attempt has already been made // and that we are actually exercising the retry code. collectData := func(ch chan string) { ln, err := net.Listen("tcp", "localhost:9125") if err != nil { ch <- err.Error() return } conn, err := ln.Accept() if err != nil { ch <- err.Error() return } b := make([]byte, 1000) n, _ := conn.Read(b) ch <- string(b[0:n]) conn.Close() ln.Close() } ch := make(chan string, 1) // don't block on put go collectData(ch) result := <-ch c.Expect(result, gs.Equals, string(matchBytes)) close(inChan) wg.Wait() // wait for output to finish shutting down }) }) }
func AMQPPluginSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) // Our two user/conn waitgroups ug := new(sync.WaitGroup) cg := new(sync.WaitGroup) // Setup the mock channel mch := NewMockAMQPChannel(ctrl) // Setup the mock amqpHub with the mock chan return aqh := NewMockAMQPConnectionHub(ctrl) aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil) errChan := make(chan error, 1) c.Specify("An amqp input", func() { // Setup all the mock calls for Init mch.EXPECT().ExchangeDeclare("", "", false, true, false, false, gomock.Any()).Return(nil) mch.EXPECT().QueueDeclare("", false, true, false, false, gomock.Any()).Return(amqp.Queue{}, nil) mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil) mch.EXPECT().Qos(2, 0, false).Return(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // set up mock helper, decoder set, and packSupply channel ith.MockHelper = NewMockPluginHelper(ctrl) ith.MockInputRunner = NewMockInputRunner(ctrl) mockDRunner := NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) amqpInput := new(AMQPInput) amqpInput.amqpHub = aqh config := amqpInput.ConfigStruct().(*AMQPInputConfig) config.URL = "" config.Exchange = "" config.ExchangeType = "" config.RoutingKey = "test" config.QueueTTL = 300000 c.Specify("with a valid setup and no decoder", func() { err := amqpInput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpInput.ch, gs.Equals, mch) c.Specify("consumes a message", func() { // Create a channel to send data to the input // Drop a message on there and close the channel streamChan := make(chan amqp.Delivery, 1) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "text/plain", Body: []byte("This is a message"), Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Expect the injected packet ith.MockInputRunner.EXPECT().Inject(gomock.Any()) // Increase the usage since Run decrements it on close ug.Add(1) ith.PackSupply <- ith.Pack go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() ith.PackSupply <- ith.Pack close(streamChan) err = <-errChan c.Expect(err, gs.IsNil) c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp") c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message") }) }) c.Specify("with a valid setup using a decoder", func() { decoderName := "defaultDecoder" config.Decoder = decoderName err := amqpInput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpInput.ch, gs.Equals, mch) // Mock up our default decoder runner and decoder. ith.MockInputRunner.EXPECT().Name().Return("AMQPInput") decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder") decCall.Return(mockDRunner, true) mockDecoder := NewMockDecoder(ctrl) mockDRunner.EXPECT().Decoder().Return(mockDecoder) c.Specify("consumes a message", func() { packs := []*PipelinePack{ith.Pack} mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil) // Create a channel to send data to the input // Drop a message on there and close the channel streamChan := make(chan amqp.Delivery, 1) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "text/plain", Body: []byte("This is a message"), Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Expect the injected packet ith.MockInputRunner.EXPECT().Inject(gomock.Any()) // Increase the usage since Run decrements it on close ug.Add(1) ith.PackSupply <- ith.Pack go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() ith.PackSupply <- ith.Pack close(streamChan) err = <-errChan c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp") c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message") }) c.Specify("consumes a serialized message", func() { encoder := client.NewProtobufEncoder(nil) streamChan := make(chan amqp.Delivery, 1) msg := new(message.Message) msg.SetUuid(uuid.NewRandom()) msg.SetTimestamp(time.Now().UnixNano()) msg.SetType("logfile") msg.SetLogger("/a/nice/path") msg.SetSeverity(int32(0)) msg.SetEnvVersion("0.2") msg.SetPid(0) msg.SetPayload("This is a message") msg.SetHostname("TestHost") msgBody := make([]byte, 0, 500) _ = encoder.EncodeMessageStream(msg, &msgBody) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "application/hekad", Body: msgBody, Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Expect the decoded pack mockDRunner.EXPECT().InChan().Return(ith.DecodeChan) // Increase the usage since Run decrements it on close ug.Add(1) ith.PackSupply <- ith.Pack go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() packRef := <-ith.DecodeChan c.Expect(ith.Pack, gs.Equals, packRef) // Ignore leading 5 bytes of encoded message as thats the header c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:])) ith.PackSupply <- ith.Pack close(streamChan) err = <-errChan c.Expect(err, gs.IsNil) }) }) }) c.Specify("An amqp output", func() { oth := plugins_ts.NewOutputTestHelper(ctrl) pConfig := NewPipelineConfig(nil) amqpOutput := new(AMQPOutput) amqpOutput.amqpHub = aqh config := amqpOutput.ConfigStruct().(*AMQPOutputConfig) config.URL = "" config.Exchange = "" config.ExchangeType = "" config.RoutingKey = "test" closeChan := make(chan *amqp.Error) inChan := make(chan *PipelinePack, 1) mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan) mch.EXPECT().ExchangeDeclare("", "", false, true, false, false, gomock.Any()).Return(nil) // Increase the usage since Run decrements it on close ug.Add(1) // Expect the close and the InChan calls aqh.EXPECT().Close("", cg) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true c.Specify("publishes a plain message", func() { encoder := new(plugins.PayloadEncoder) econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig) econfig.AppendNewlines = false encoder.Init(econfig) payloadBytes, err := encoder.Encode(pack) config.Encoder = "PayloadEncoder" config.ContentType = "text/plain" oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil) err = amqpOutput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpOutput.ch, gs.Equals, mch) mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil) inChan <- pack close(inChan) close(closeChan) go func() { err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() ug.Wait() err = <-errChan c.Expect(err, gs.IsNil) }) c.Specify("publishes a serialized message", func() { encoder := new(ProtobufEncoder) encoder.SetPipelineConfig(pConfig) encoder.Init(nil) protoBytes, err := encoder.Encode(pack) c.Expect(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil) err = amqpOutput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpOutput.ch, gs.Equals, mch) mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil) inChan <- pack close(inChan) close(closeChan) go func() { err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() ug.Wait() err = <-errChan c.Expect(err, gs.IsNil) }) }) }
func IrcOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() pipelineConfig := NewPipelineConfig(nil) outTestHelper := plugins_ts.NewOutputTestHelper(ctrl) var wg sync.WaitGroup errChan := make(chan error, 1) tickChan := make(chan time.Time) inChan := make(chan *PipelinePack, 5) c.Specify("An IrcOutput", func() { ircOutput := new(IrcOutput) ircOutput.InitIrcCon = NewMockIrcConn encoder := new(plugins.PayloadEncoder) encoder.Init(&plugins.PayloadEncoderConfig{}) config := ircOutput.ConfigStruct().(*IrcOutputConfig) config.Server = "irc.example.org" config.Nick = "heka_bot" config.Ident = "heka" config.Channels = []string{"#test_channel"} msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pipelineConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true c.Specify("requires an encoder", func() { err := ircOutput.Init(config) c.Assume(err, gs.IsNil) outTestHelper.MockOutputRunner.EXPECT().Encoder().Return(nil) err = ircOutput.Run(outTestHelper.MockOutputRunner, outTestHelper.MockHelper) c.Expect(err, gs.Not(gs.IsNil)) }) c.Specify("that is started", func() { outTestHelper.MockOutputRunner.EXPECT().Ticker().Return(tickChan) outTestHelper.MockOutputRunner.EXPECT().Encoder().Return(encoder) outTestHelper.MockOutputRunner.EXPECT().InChan().Return(inChan) outTestHelper.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes() startOutput := func() { wg.Add(1) go func() { err := ircOutput.Run(outTestHelper.MockOutputRunner, outTestHelper.MockHelper) errChan <- err wg.Done() }() <-mockIrcConn.connected } c.Specify("sends incoming messages to an irc server", func() { err := ircOutput.Init(config) c.Assume(err, gs.IsNil) c.Assume(len(ircOutput.Channels), gs.Equals, 1) ircChan := ircOutput.Channels[0] startOutput() // Send the data inChan <- pack // wait for it to arrive p := <-ircOutput.OutQueue ircOutput.OutQueue <- p // once it's arrived send a tick so it gets processed tickChan <- time.Now() close(inChan) wg.Wait() // Verify we've exited c.Expect(mockIrcConn.quit, gs.IsTrue) // verify the message was sent msgs := mockIrcConn.msgs[ircChan] c.Expect(len(msgs), gs.Equals, 1) c.Expect(msgs[0], gs.Equals, string(*msg.Payload)) }) c.Specify("drops messages when outqueue is full", func() { config.QueueSize = 1 err := ircOutput.Init(config) c.Assume(err, gs.IsNil) c.Assume(len(ircOutput.Channels), gs.Equals, 1) ircChan := ircOutput.Channels[0] outTestHelper.MockOutputRunner.EXPECT().LogError( ErrOutQueueFull) outTestHelper.MockOutputRunner.EXPECT().LogError( fmt.Errorf("%s Channel: %s.", ErrBacklogQueueFull, ircChan)) startOutput() // Need to wait for callbacks to get registered before calling part. // If we've called connected, then callbacks have been registered. // Leave the irc channel so we can fill up the backlog ircOutput.Conn.Part(ircChan) c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED) // Put some data into the inChan inChan <- pack // Wait for it to arrive in the OutQueue and put it back // We *must* do this before sending the tick to avoid data races p := <-ircOutput.OutQueue ircOutput.OutQueue <- p // One tick so that it goes from the OutQueue into the BacklogQueue tickChan <- time.Now() // Verify the item is in the backlog queue and put it back // (It will never leave this loop if the msg never arrives) queue := ircOutput.BacklogQueues[0] msg := <-queue queue <- msg c.Expect(len(queue), gs.Equals, 1) // Send another message to fill the OutQueue inChan <- pack // Wait for it to arrive, again p = <-ircOutput.OutQueue ircOutput.OutQueue <- p // Send the tick after it's in the OutQueue so we can try processing // it. This is where we drop it since we cant send, and the // BacklogQueue is already full. tickChan <- time.Now() // Now we want to also cause the OutQueue to drop a message. // We don't have to wait for it to arrive in the OutQueue like we do above // since it shouldn't make it there. inChan <- pack inChan <- pack close(inChan) wg.Wait() // Verify we've exited c.Expect(mockIrcConn.quit, gs.IsTrue) // verify the backlog queue is empty c.Expect(len(queue), gs.Equals, 0) // verify that no messages were sent. msgs := mockIrcConn.msgs[ircChan] c.Expect(len(msgs), gs.Equals, 0) }) c.Specify("automatically reconnects on disconnect", func() { config.TimeBeforeReconnect = 0 err := ircOutput.Init(config) c.Assume(err, gs.IsNil) outTestHelper.MockOutputRunner.EXPECT().LogMessage(DisconnectMsg) outTestHelper.MockOutputRunner.EXPECT().LogMessage(ReconnectedMsg) startOutput() ircOutput.Conn.Disconnect() // verify we've been disconnected c.Expect(<-mockIrcConn.connected, gs.IsFalse) // Wait to become reconnected. c.Expect(<-mockIrcConn.connected, gs.IsTrue) close(inChan) wg.Wait() c.Expect(mockIrcConn.quit, gs.IsTrue) }) c.Specify("logs an error and exits from Run() when it cannot reconnect", func() { config.TimeBeforeReconnect = 0 err := ircOutput.Init(config) c.Assume(err, gs.IsNil) // Explicitly fail to reconnect mockIrcConn.failReconnect = true outTestHelper.MockOutputRunner.EXPECT().LogMessage(DisconnectMsg) outTestHelper.MockOutputRunner.EXPECT().LogError(fmt.Errorf(ErrReconnecting, mockErrFailReconnect)) startOutput() // Need to wait for callbacks to get registered before we disconnect ircOutput.Conn.Disconnect() // verify we've been disconnected // Run() should return without closing the inChan because the plugin // automatically cleans up and returns if it fails to reconnect wg.Wait() c.Expect(mockIrcConn.quit, gs.IsTrue) // do this at the end for test cleanup purposes close(inChan) }) c.Specify("when kicked from an irc channel", func() { c.Specify("rejoins when configured to do so", func() { config.RejoinOnKick = true err := ircOutput.Init(config) c.Assume(err, gs.IsNil) c.Assume(len(ircOutput.Channels), gs.Equals, 1) ircChan := ircOutput.Channels[0] startOutput() // trigger a kick event for the irc channel kick(mockIrcConn, ircChan) c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED) // quit close(inChan) }) c.Specify("doesnt rejoin when it isnt configured to", func() { config.RejoinOnKick = false // this is already the default err := ircOutput.Init(config) c.Assume(err, gs.IsNil) c.Assume(len(ircOutput.Channels), gs.Equals, 1) ircChan := ircOutput.Channels[0] startOutput() // Since this is the only channel we're in, we should get an // error that there are no channels to join left after being // kicked outTestHelper.MockOutputRunner.EXPECT().LogError(ErrNoJoinableChannels) // trigger a kick for the irc channel kick(mockIrcConn, ircChan) c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN) // We shouldnt need to close the inChan, since we have no // joinable channels, we should be cleaning up already. }) wg.Wait() }) c.Specify("when trying to join an unjoinable channel", func() { c.Specify("it logs an error", func() { config.Channels = []string{"foo", "baz"} err := ircOutput.Init(config) c.Assume(err, gs.IsNil) c.Assume(len(ircOutput.Channels), gs.Equals, 2) ircChan := ircOutput.Channels[0] startOutput() c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED) reason := "foo" err = IrcCannotJoinError{ircChan, reason} outTestHelper.MockOutputRunner.EXPECT().LogError(err) args := []string{"", ircChan, reason} event := irc.Event{Arguments: args} c.Specify("when the channel key is wrong", func() { event.Code = IRC_ERR_BADCHANNELKEY }) c.Specify("when banned from the channel", func() { event.Code = IRC_ERR_BANNEDFROMCHAN }) c.Specify("when there is no such channel", func() { event.Code = IRC_NOSUCHCHANNEL }) c.Specify("when the channel is invite only", func() { event.Code = IRC_ERR_INVITEONLYCHAN }) ircOutput.Conn.RunCallbacks(&event) c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN) // should still be unable to join ircOutput.Join(ircChan) c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN) close(inChan) }) c.Specify("it exits if there are no joinable channels", func() { err := ircOutput.Init(config) c.Assume(err, gs.IsNil) c.Assume(len(ircOutput.Channels), gs.Equals, 1) ircChan := ircOutput.Channels[0] startOutput() c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED) reason := "foo" err = IrcCannotJoinError{ircChan, reason} outTestHelper.MockOutputRunner.EXPECT().LogError(err) outTestHelper.MockOutputRunner.EXPECT().LogError(ErrNoJoinableChannels) args := []string{"", ircChan, reason} event := irc.Event{Arguments: args, Code: IRC_ERR_BADCHANNELKEY} ircOutput.Conn.RunCallbacks(&event) // No close since we should exit without. }) wg.Wait() }) c.Specify("when trying to join a full channel", func() { config.TimeBeforeRejoin = 0 err := ircOutput.Init(config) c.Assume(err, gs.IsNil) c.Assume(len(ircOutput.Channels), gs.Equals, 1) ircChan := ircOutput.Channels[0] startOutput() c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED) // We should be able to join before this ircOutput.Join(ircChan) reason := "full channel" err = fmt.Errorf("%s. Retrying in %d seconds.", IrcCannotJoinError{ircChan, reason}.Error(), ircOutput.TimeBeforeRejoin) args := []string{"", ircChan, reason} event := irc.Event{Code: IRC_ERR_CHANNELISFULL, Arguments: args} c.Specify("logs an error after using up its attempts", func() { mockIrcConn.failJoin = true // We should leave the channel so that we aren't just // causing a failure to join even when we're already in the // channel ircOutput.Conn.Part(ircChan) maxRetries := int(ircOutput.MaxJoinRetries) c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED) gomock.InOrder( outTestHelper.MockOutputRunner.EXPECT().LogError( err).Times(maxRetries), outTestHelper.MockOutputRunner.EXPECT().LogError( fmt.Errorf(ErrUsedUpRetryAttempts, ircChan)), outTestHelper.MockOutputRunner.EXPECT().LogError( ErrNoJoinableChannels), ) for i := 1; i <= maxRetries; i++ { // Trigger the event the number of times ircOutput.Conn.RunCallbacks(&event) c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, i) } c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED) ircOutput.Conn.RunCallbacks(&event) // We've used up our attempts, c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN) ircOutput.Conn.RunCallbacks(&event) c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN) }) c.Specify("resets its attempts when it successful", func() { mockIrcConn.failJoin = true outTestHelper.MockOutputRunner.EXPECT().LogError(err).Times(2) ircOutput.Conn.RunCallbacks(&event) c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, 1) mockIrcConn.failJoin = false ircOutput.Conn.RunCallbacks(&event) c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, 0) }) close(inChan) wg.Wait() }) c.Specify("when banned from a server logs an error and quits", func() { err := ircOutput.Init(config) c.Assume(err, gs.IsNil) startOutput() event := irc.Event{Code: IRC_ERR_YOUREBANNEDCREEP} outTestHelper.MockOutputRunner.EXPECT().LogError(ErrBannedFromServer) ircOutput.Conn.RunCallbacks(&event) // We should be able to exit without closing the channel first wg.Wait() // Clean this up close(inChan) }) c.Expect(<-errChan, gs.IsNil) c.Expect(<-mockIrcConn.connected, gs.IsFalse) }) // Cleanup which should happen each run close(mockIrcConn.connected) close(mockIrcConn.delivered) close(mockIrcConn.Error) }) }
func FileOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano()) tmpFilePath := filepath.Join(os.TempDir(), tmpFileName) defer func() { ctrl.Finish() os.Remove(tmpFilePath) }() oth := plugins_ts.NewOutputTestHelper(ctrl) var wg sync.WaitGroup inChan := make(chan *PipelinePack, 1) pConfig := NewPipelineConfig(nil) c.Specify("A FileOutput", func() { fileOutput := new(FileOutput) encoder := new(plugins.PayloadEncoder) encoder.Init(encoder.ConfigStruct()) config := fileOutput.ConfigStruct().(*FileOutputConfig) config.Path = tmpFilePath msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true c.Specify("w/ ProtobufEncoder", func() { encoder := new(ProtobufEncoder) encoder.Init(nil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) c.Specify("uses framing", func() { oth.MockOutputRunner.EXPECT().SetUseFraming(true) err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) wg.Add(1) go func() { err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() close(inChan) wg.Wait() }) c.Specify("but not if config says not to", func() { useFraming := false config.UseFraming = &useFraming err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) wg.Add(1) go func() { err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() close(inChan) wg.Wait() // We should fail if SetUseFraming is called since we didn't // EXPECT it. }) }) c.Specify("processes incoming messages", func() { err := fileOutput.Init(config) c.Assume(err, gs.IsNil) fileOutput.file.Close() // Save for comparison. payload := fmt.Sprintf("%s\n", pack.Message.GetPayload()) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) wg.Add(1) go fileOutput.receiver(oth.MockOutputRunner, &wg) inChan <- pack close(inChan) outBatch := <-fileOutput.batchChan wg.Wait() c.Expect(string(outBatch), gs.Equals, payload) }) c.Specify("commits to a file", func() { outStr := "Write me out to the log file" outBytes := []byte(outStr) c.Specify("with default settings", func() { err := fileOutput.Init(config) c.Assume(err, gs.IsNil) // Start committer loop wg.Add(1) go fileOutput.committer(oth.MockOutputRunner, &wg) // Feed and close the batchChan go func() { fileOutput.batchChan <- outBytes _ = <-fileOutput.backChan // clear backChan to prevent blocking close(fileOutput.batchChan) }() wg.Wait() // Wait for the file close operation to happen. //for ; err == nil; _, err = fileOutput.file.Stat() { //} tmpFile, err := os.Open(tmpFilePath) defer tmpFile.Close() c.Assume(err, gs.IsNil) contents, err := ioutil.ReadAll(tmpFile) c.Assume(err, gs.IsNil) c.Expect(string(contents), gs.Equals, outStr) }) c.Specify("with different Perm settings", func() { config.Perm = "600" err := fileOutput.Init(config) c.Assume(err, gs.IsNil) // Start committer loop wg.Add(1) go fileOutput.committer(oth.MockOutputRunner, &wg) // Feed and close the batchChan go func() { fileOutput.batchChan <- outBytes _ = <-fileOutput.backChan // clear backChan to prevent blocking close(fileOutput.batchChan) }() wg.Wait() // Wait for the file close operation to happen. //for ; err == nil; _, err = fileOutput.file.Stat() { //} tmpFile, err := os.Open(tmpFilePath) defer tmpFile.Close() c.Assume(err, gs.IsNil) fileInfo, err := tmpFile.Stat() c.Assume(err, gs.IsNil) fileMode := fileInfo.Mode() if runtime.GOOS == "windows" { c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-") } else { // 7 consecutive dashes implies no perms for group or other c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------") } }) }) if runtime.GOOS != "windows" { c.Specify("Init halts if basedirectory is not writable", func() { tmpdir := filepath.Join(os.TempDir(), "tmpdir") err := os.MkdirAll(tmpdir, 0400) c.Assume(err, gs.IsNil) config.Path = filepath.Join(tmpdir, "out.txt") err = fileOutput.Init(config) c.Assume(err, gs.Not(gs.IsNil)) os.RemoveAll(tmpdir) }) c.Specify("honors folder_perm setting", func() { config.FolderPerm = "750" subdir := filepath.Join(os.TempDir(), "subdir") config.Path = filepath.Join(subdir, "out.txt") err := fileOutput.Init(config) defer os.RemoveAll(subdir) c.Assume(err, gs.IsNil) fi, err := os.Stat(subdir) c.Expect(fi.IsDir(), gs.IsTrue) c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750)) }) } c.Specify("that starts receiving w/ a flush interval", func() { config.FlushInterval = 100000000 // We'll trigger the timer manually. inChan := make(chan *PipelinePack) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) timerChan := make(chan time.Time) msg2 := pipeline_ts.GetTestMessage() pack2 := NewPipelinePack(pConfig.InputRecycleChan()) pack2.Message = msg2 recvWithConfig := func(config *FileOutputConfig) { err := fileOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go fileOutput.receiver(oth.MockOutputRunner, &wg) runtime.Gosched() // Yield so we can overwrite the timerChan. fileOutput.timerChan = timerChan } cleanUp := func() { close(inChan) fileOutput.file.Close() wg.Done() } c.Specify("honors flush interval", func() { oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) recvWithConfig(config) defer cleanUp() inChan <- pack select { case _ = <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") default: } timerChan <- time.Now() select { case _ = <-fileOutput.batchChan: default: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("honors flush interval AND flush count", func() { oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2)) config.FlushCount = 2 recvWithConfig(config) defer cleanUp() inChan <- pack select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") default: } timerChan <- time.Now() select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") default: } inChan <- pack2 runtime.Gosched() select { case <-fileOutput.batchChan: default: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("honors flush interval OR flush count", func() { oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack)) config.FlushCount = 2 config.FlushOperator = "OR" recvWithConfig(config) defer cleanUp() inChan <- pack select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") default: } c.Specify("when interval triggers first", func() { timerChan <- time.Now() select { case <-fileOutput.batchChan: default: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("when count triggers first", func() { out, err := encoder.Encode(pack2) oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err) inChan <- pack2 runtime.Gosched() select { case <-fileOutput.batchChan: default: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) }) }) }) }
func AMQPPluginSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) // Our two user/conn waitgroups. ug := new(sync.WaitGroup) cg := new(sync.WaitGroup) // Setup the mock channel. mch := NewMockAMQPChannel(ctrl) // Setup the mock amqpHub with the mock chan return. aqh := NewMockAMQPConnectionHub(ctrl) aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil) errChan := make(chan error, 1) bytesChan := make(chan []byte, 1) c.Specify("An amqp input", func() { // Setup all the mock calls for Init. mch.EXPECT().ExchangeDeclare("", "", false, true, false, false, gomock.Any()).Return(nil) mch.EXPECT().QueueDeclare("", false, true, false, false, gomock.Any()).Return(amqp.Queue{}, nil) mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil) mch.EXPECT().Qos(2, 0, false).Return(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // Set up relevant mocks. ith.MockHelper = NewMockPluginHelper(ctrl) ith.MockInputRunner = NewMockInputRunner(ctrl) ith.MockSplitterRunner = NewMockSplitterRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner) amqpInput := new(AMQPInput) amqpInput.amqpHub = aqh config := amqpInput.ConfigStruct().(*AMQPInputConfig) config.URL = "" config.Exchange = "" config.ExchangeType = "" config.RoutingKey = "test" config.QueueTTL = 300000 err := amqpInput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpInput.ch, gs.Equals, mch) c.Specify("consumes a text message", func() { // Create a channel to send data to the input. Drop a message on // there and close the channel. streamChan := make(chan amqp.Delivery, 1) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "text/plain", Body: []byte("This is a message"), Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Increase the usage since Run decrements it on close. ug.Add(1) splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil) splitCall.Do(func(recd []byte, del Deliverer) { bytesChan <- recd }) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() msgBytes := <-bytesChan c.Expect(string(msgBytes), gs.Equals, "This is a message") close(streamChan) err = <-errChan }) c.Specify("consumes a protobuf encoded message", func() { encoder := client.NewProtobufEncoder(nil) streamChan := make(chan amqp.Delivery, 1) msg := new(message.Message) msg.SetUuid(uuid.NewRandom()) msg.SetTimestamp(time.Now().UnixNano()) msg.SetType("logfile") msg.SetLogger("/a/nice/path") msg.SetSeverity(int32(0)) msg.SetEnvVersion("0.2") msg.SetPid(0) msg.SetPayload("This is a message") msg.SetHostname("TestHost") msgBody := make([]byte, 0, 500) _ = encoder.EncodeMessageStream(msg, &msgBody) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "application/hekad", Body: msgBody, Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Increase the usage since Run decrements it on close. ug.Add(1) splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil) splitCall.Do(func(recd []byte, del Deliverer) { bytesChan <- recd }) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true) go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() msgBytes := <-bytesChan c.Expect(string(msgBytes), gs.Equals, string(msgBody)) close(streamChan) err = <-errChan c.Expect(err, gs.IsNil) }) }) c.Specify("An amqp output", func() { oth := plugins_ts.NewOutputTestHelper(ctrl) pConfig := NewPipelineConfig(nil) amqpOutput := new(AMQPOutput) amqpOutput.amqpHub = aqh config := amqpOutput.ConfigStruct().(*AMQPOutputConfig) config.URL = "" config.Exchange = "" config.ExchangeType = "" config.RoutingKey = "test" closeChan := make(chan *amqp.Error) inChan := make(chan *PipelinePack, 1) mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan) mch.EXPECT().ExchangeDeclare("", "", false, true, false, false, gomock.Any()).Return(nil) // Increase the usage since Run decrements it on close. ug.Add(1) // Expect the close and the InChan calls. aqh.EXPECT().Close("", cg) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true c.Specify("publishes a plain message", func() { encoder := new(plugins.PayloadEncoder) econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig) econfig.AppendNewlines = false encoder.Init(econfig) payloadBytes, err := encoder.Encode(pack) config.Encoder = "PayloadEncoder" config.ContentType = "text/plain" oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil) err = amqpOutput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpOutput.ch, gs.Equals, mch) mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil) inChan <- pack close(inChan) close(closeChan) go func() { err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() ug.Wait() err = <-errChan c.Expect(err, gs.IsNil) }) c.Specify("publishes a serialized message", func() { encoder := new(ProtobufEncoder) encoder.SetPipelineConfig(pConfig) encoder.Init(nil) protoBytes, err := encoder.Encode(pack) c.Expect(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil) err = amqpOutput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpOutput.ch, gs.Equals, mch) mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil) inChan <- pack close(inChan) close(closeChan) go func() { err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() ug.Wait() err = <-errChan c.Expect(err, gs.IsNil) }) }) }
func TcpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests-") os.MkdirAll(tmpDir, 0777) defer func() { ctrl.Finish() tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() pConfig := NewPipelineConfig(nil) c.Specify("TcpOutput Internals", func() { tcpOutput := new(TcpOutput) tcpOutput.connection = pipeline_ts.NewMockConn(ctrl) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true c.Specify("SetName", func() { tcpOutput.SetName("this-is a test") c.Expect(tcpOutput.name, gs.Equals, "this_is_a_test") }) c.Specify("fileExists", func() { c.Expect(fileExists(tmpDir), gs.IsTrue) c.Expect(fileExists(filepath.Join(tmpDir, "test.log")), gs.IsFalse) }) c.Specify("extractBufferId", func() { id, err := extractBufferId("555.log") c.Expect(err, gs.IsNil) c.Expect(id, gs.Equals, uint(555)) id, err = extractBufferId("") c.Expect(err, gs.Not(gs.IsNil)) id, err = extractBufferId("a.log") c.Expect(err, gs.Not(gs.IsNil)) }) c.Specify("findBufferId", func() { c.Expect(findBufferId(tmpDir, true), gs.Equals, uint(0)) c.Expect(findBufferId(tmpDir, false), gs.Equals, uint(0)) _, err := os.OpenFile(filepath.Join(tmpDir, "4.log"), os.O_CREATE, 0644) c.Expect(err, gs.IsNil) _, err = os.OpenFile(filepath.Join(tmpDir, "5.log"), os.O_CREATE, 0644) c.Expect(err, gs.IsNil) _, err = os.OpenFile(filepath.Join(tmpDir, "6a.log"), os.O_CREATE, 0644) c.Expect(err, gs.IsNil) c.Expect(findBufferId(tmpDir, false), gs.Equals, uint(4)) c.Expect(findBufferId(tmpDir, true), gs.Equals, uint(5)) }) c.Specify("writeCheckpoint", func() { tcpOutput.checkpointFilename = filepath.Join(tmpDir, "cp.txt") err := tcpOutput.writeCheckpoint(43, 99999) c.Expect(err, gs.IsNil) c.Expect(fileExists(tcpOutput.checkpointFilename), gs.IsTrue) id, offset, err := readCheckpoint(tcpOutput.checkpointFilename) c.Expect(err, gs.IsNil) c.Expect(id, gs.Equals, uint(43)) c.Expect(offset, gs.Equals, int64(99999)) err = tcpOutput.writeCheckpoint(43, 1) c.Expect(err, gs.IsNil) id, offset, err = readCheckpoint(tcpOutput.checkpointFilename) c.Expect(err, gs.IsNil) c.Expect(id, gs.Equals, uint(43)) c.Expect(offset, gs.Equals, int64(1)) tcpOutput.checkpointFile.Close() }) c.Specify("readCheckpoint", func() { cp := filepath.Join(tmpDir, "cp.txt") file, err := os.OpenFile(cp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) c.Expect(err, gs.IsNil) id, offset, err := readCheckpoint(cp) c.Expect(err, gs.Not(gs.IsNil)) file.WriteString("22") id, offset, err = readCheckpoint(cp) c.Expect(err.Error(), gs.Equals, "invalid checkpoint format") file.Seek(0, 0) file.WriteString("aa 22") id, offset, err = readCheckpoint(cp) c.Expect(err.Error(), gs.Equals, "invalid checkpoint id") file.Seek(0, 0) file.WriteString("43 aa") id, offset, err = readCheckpoint(cp) c.Expect(err.Error(), gs.Equals, "invalid checkpoint offset") file.Seek(0, 0) file.WriteString("43 22") file.Close() id, offset, err = readCheckpoint(cp) c.Expect(err, gs.IsNil) c.Expect(id, gs.Equals, uint(43)) c.Expect(offset, gs.Equals, int64(22)) }) c.Specify("writeToNextFile", func() { tcpOutput.checkpointFilename = filepath.Join(tmpDir, "cp.txt") tcpOutput.queue = tmpDir err := tcpOutput.writeToNextFile() c.Expect(err, gs.IsNil) c.Expect(fileExists(getQueueFilename(tcpOutput.queue, tcpOutput.writeId)), gs.IsTrue) tcpOutput.writeFile.WriteString("this is a test item") tcpOutput.writeFile.Close() tcpOutput.writeCheckpoint(tcpOutput.writeId, 10) tcpOutput.checkpointFile.Close() err = tcpOutput.readFromNextFile() buf := make([]byte, 4) n, err := tcpOutput.readFile.Read(buf) c.Expect(n, gs.Equals, 4) c.Expect("test", gs.Equals, string(buf)) tcpOutput.writeFile.Close() tcpOutput.readFile.Close() }) }) c.Specify("TcpOutput", func() { origBaseDir := Globals().BaseDir Globals().BaseDir = tmpDir defer func() { Globals().BaseDir = origBaseDir }() tcpOutput := new(TcpOutput) tcpOutput.SetName("test") config := tcpOutput.ConfigStruct().(*TcpOutputConfig) tcpOutput.Init(config) tickChan := make(chan time.Time) oth := plugins_ts.NewOutputTestHelper(ctrl) oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan) var wg sync.WaitGroup inChan := make(chan *PipelinePack, 1) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true c.Specify("writes out to the network", func() { inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() inChanCall.Return(inChan) collectData := func(ch chan string) { ln, err := net.Listen("tcp", "localhost:9125") if err != nil { ch <- err.Error() return } ch <- "ready" conn, err := ln.Accept() if err != nil { ch <- err.Error() return } b := make([]byte, 1000) n, _ := conn.Read(b) ch <- string(b[0:n]) conn.Close() ln.Close() } ch := make(chan string, 1) // don't block on put go collectData(ch) result := <-ch // wait for server err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) outStr := "Write me out to the network" pack.Message.SetPayload(outStr) go func() { wg.Add(1) c.Expect(err, gs.IsNil) err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper) wg.Done() }() inChan <- pack output := false for x := 0; x < 5; x++ { if fileExists(tcpOutput.checkpointFilename) { output = true break } time.Sleep(time.Duration(500) * time.Millisecond) } c.Expect(output, gs.Equals, true) close(inChan) wg.Wait() // wait for close to finish, prevents intermittent test failures matchBytes := make([]byte, 0, 1000) newpack := NewPipelinePack(nil) newpack.Message = msg newpack.Decoded = true newpack.Message.SetPayload(outStr) err = ProtobufEncodeMessage(newpack, &matchBytes) c.Expect(err, gs.IsNil) result = <-ch c.Expect(result, gs.Equals, string(matchBytes)) id, offset, err := readCheckpoint(tcpOutput.checkpointFilename) c.Expect(err, gs.IsNil) c.Expect(id, gs.Equals, uint(1)) c.Expect(offset, gs.Equals, int64(115)) }) c.Specify("far end not initially listening", func() { inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() inChanCall.Return(inChan) oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) outStr := "Write me out to the network" pack.Message.SetPayload(outStr) go func() { wg.Add(1) c.Expect(err, gs.IsNil) err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper) wg.Done() }() inChan <- pack output := false for x := 0; x < 5; x++ { if fileExists(getQueueFilename(tcpOutput.queue, tcpOutput.writeId)) { output = true time.Sleep(time.Duration(1) * time.Second) // give the send time to fail break } time.Sleep(time.Duration(500) * time.Millisecond) } c.Expect(output, gs.Equals, true) collectData := func(ch chan string) { ln, err := net.Listen("tcp", "localhost:9125") if err != nil { ch <- err.Error() return } conn, err := ln.Accept() if err != nil { ch <- err.Error() return } b := make([]byte, 1000) n, _ := conn.Read(b) ch <- string(b[0:n]) conn.Close() ln.Close() } ch := make(chan string, 1) // don't block on put go collectData(ch) result := <-ch // wait for server close(inChan) wg.Wait() // wait for close to finish, prevents intermittent test failures matchBytes := make([]byte, 0, 1000) newpack := NewPipelinePack(nil) newpack.Message = msg newpack.Decoded = true newpack.Message.SetPayload(outStr) err = ProtobufEncodeMessage(newpack, &matchBytes) c.Expect(err, gs.IsNil) c.Expect(result, gs.Equals, string(matchBytes)) id, offset, err := readCheckpoint(tcpOutput.checkpointFilename) c.Expect(err, gs.IsNil) c.Expect(id, gs.Equals, uint(1)) c.Expect(offset, gs.Equals, int64(115)) }) }) }
func UnixOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) c.Specify("A UnixOutput", func() { unixOutput := new(UdpOutput) config := unixOutput.ConfigStruct().(*UdpOutputConfig) oth := plugins_ts.NewOutputTestHelper(ctrl) encoder := new(plugins.PayloadEncoder) encoder.Init(new(plugins.PayloadEncoderConfig)) inChan := make(chan *pipeline.PipelinePack, 1) rChan := make(chan *pipeline.PipelinePack, 1) msg := pipeline_ts.GetTestMessage() payload := "Write me out to the network." msg.SetPayload(payload) pack := pipeline.NewPipelinePack(rChan) pack.Message = msg oth.MockOutputRunner.EXPECT().InChan().Return(inChan) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) c.Specify("using UDP", func() { addr := "@/heka/unix-out" config.Address = addr ch := make(chan string, 1) var wg sync.WaitGroup var rAddr net.Addr collectData := func() { conn, err := net.ListenPacket("unix", addr) if err != nil { ch <- err.Error() return } ch <- "ready" b := make([]byte, 1000) var n int n, rAddr, _ = conn.ReadFrom(b) ch <- string(b[:n]) conn.Close() } go collectData() result := <-ch // Wait for server to be ready. c.Assume(result, gs.Equals, "ready") c.Specify("writes out to the network", func() { err := unixOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() inChan <- pack result = <-ch c.Expect(result, gs.Equals, payload) close(inChan) wg.Wait() }) c.Specify("uses the specified local address", func() { config.LocalAddress = "@/heka/unix-out" err := unixOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go func() { err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() inChan <- pack result = <-ch c.Expect(result, gs.Equals, payload) c.Expect(rAddr.Network(), gs.Equals, "unix") c.Expect(rAddr.String(), gs.Equals, "@/heka/unix-out") close(inChan) wg.Wait() }) }) }) }
func NsqOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := pipeline.NewPipelineConfig(nil) var wg sync.WaitGroup errChan := make(chan error, 1) defer close(errChan) c.Specify("A nsq output", func() { output := new(NsqOutput) oth := plugins_ts.NewOutputTestHelper(ctrl) config := output.ConfigStruct().(*NsqOutputConfig) config.Topic = "test" startOutput := func() { wg.Add(1) go func() { errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper) wg.Done() }() } var mockProducer *MockProducer output.newProducer = func(addr string, config *nsq.Config) (Producer, error) { mockProducer, _ = NewMockProducer(addr, config) return mockProducer, nil } msg := pipeline_ts.GetTestMessage() pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg c.Specify("requires at least one address", func() { err := output.Init(config) c.Expect(err, gs.Not(gs.IsNil)) }) config.Addresses = []string{"test.com:4150"} c.Specify("requires an encoder", func() { err := output.Init(config) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(nil) err = output.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.Not(gs.IsNil)) }) c.Specify("that is started", func() { encoder := new(plugins.PayloadEncoder) encoder.Init(&plugins.PayloadEncoderConfig{PrefixTs: false}) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) inChan := make(chan *pipeline.PipelinePack, 1) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) c.Specify("publishes a message with the configured topic", func() { err := output.Init(config) c.Assume(err, gs.IsNil) startOutput() inChan <- pack transaction := <-mockProducer.respChan c.Expect(transaction.Error, gs.IsNil) close(inChan) wg.Wait() c.Expect(<-errChan, gs.IsNil) }) }) }) }
func OutputSpec(c gs.Context) { t := new(ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() oth := plugins_ts.NewOutputTestHelper(ctrl) pConfig := pipeline.NewPipelineConfig(nil) inChan := make(chan *pipeline.PipelinePack, 1) timer := make(chan time.Time, 1) c.Specify("A SandboxOutput", func() { output := new(SandboxOutput) output.SetPipelineConfig(pConfig) conf := output.ConfigStruct().(*sandbox.SandboxConfig) conf.ScriptFilename = "../lua/testsupport/output.lua" conf.ModuleDirectory = "../lua/modules;../lua/testsupport/modules" supply := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(supply) data := "1376389920 debug id=2321 url=example.com item=1" pack.Message.SetPayload(data) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes() oth.MockOutputRunner.EXPECT().Ticker().Return(timer) c.Specify("writes a payload to file", func() { err := output.Init(conf) c.Expect(err, gs.IsNil) inChan <- pack close(inChan) err = output.Run(oth.MockOutputRunner, oth.MockHelper) c.Assume(err, gs.IsNil) tmpFile, err := os.Open("output.lua.txt") defer tmpFile.Close() c.Assume(err, gs.IsNil) contents, err := ioutil.ReadAll(tmpFile) c.Assume(err, gs.IsNil) c.Expect(string(contents), gs.Equals, data) c.Expect(output.processMessageCount, gs.Equals, int64(1)) c.Expect(output.processMessageSamples, gs.Equals, int64(1)) c.Expect(output.processMessageFailures, gs.Equals, int64(0)) }) c.Specify("failure processing data", func() { err := output.Init(conf) c.Expect(err, gs.IsNil) pack.Message.SetPayload("FAILURE") pack.BufferedPack = true pack.DelivErrChan = make(chan error, 1) inChan <- pack close(inChan) err = output.Run(oth.MockOutputRunner, oth.MockHelper) c.Assume(err, gs.IsNil) c.Expect(output.processMessageFailures, gs.Equals, int64(1)) err = <-pack.DelivErrChan c.Expect(err.Error(), gs.Equals, "failure message") }) c.Specify("user abort processing data", func() { err := output.Init(conf) c.Expect(err, gs.IsNil) e := errors.New("FATAL: user abort") pack.Message.SetPayload("USERABORT") inChan <- pack err = output.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err.Error(), gs.Equals, e.Error()) }) c.Specify("fatal error processing data", func() { err := output.Init(conf) c.Expect(err, gs.IsNil) pack.Message.SetPayload("FATAL") inChan <- pack err = output.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.Not(gs.IsNil)) }) c.Specify("fatal error in timer_event", func() { err := output.Init(conf) c.Expect(err, gs.IsNil) timer <- time.Now() err = output.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.Not(gs.IsNil)) }) c.Specify("fatal error in shutdown timer_event", func() { conf.TimerEventOnShutdown = true err := output.Init(conf) c.Expect(err, gs.IsNil) close(inChan) err = output.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.Not(gs.IsNil)) }) }) }
func TestSendMessage(t *testing.T) { ctrl := gomock.NewController(t) broker := sarama.NewMockBroker(t, 2) defer func() { broker.Close() ctrl.Finish() }() topic := "test" globals := DefaultGlobals() pConfig := NewPipelineConfig(globals) broker.SetHandlerByMap(map[string]sarama.MockResponse{ "MetadataRequest": sarama.NewMockMetadataResponse(t). SetBroker(broker.Addr(), broker.BrokerID()). SetLeader(topic, 0, broker.BrokerID()), "ProduceRequest": sarama.NewMockProduceResponse(t), }) ko := new(KafkaOutput) ko.SetPipelineConfig(pConfig) config := ko.ConfigStruct().(*KafkaOutputConfig) config.Addrs = append(config.Addrs, broker.Addr()) config.Topic = topic err := ko.Init(config) if err != nil { t.Fatal(err) } oth := plugins_ts.NewOutputTestHelper(ctrl) encoder := new(plugins.PayloadEncoder) encoder.Init(encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)) inChan := make(chan *PipelinePack, 1) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() inChanCall.Return(inChan) errChan := make(chan error) startOutput := func() { go func() { err := ko.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() } oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) outStr := "Write me out to the network" pack.Message.SetPayload(outStr) startOutput() msgcount := atomic.LoadInt64(&ko.processMessageCount) if msgcount != 0 { t.Errorf("Invalid starting processMessageCount %d", msgcount) } msgcount = atomic.LoadInt64(&ko.processMessageFailures) if msgcount != 0 { t.Errorf("Invalid starting processMessageFailures %d", msgcount) } inChan <- pack close(inChan) err = <-errChan if err != nil { t.Errorf("Error running output %s", err) } msgcount = atomic.LoadInt64(&ko.processMessageCount) if msgcount != 1 { t.Errorf("Invalid ending processMessageCount %d", msgcount) } msgcount = atomic.LoadInt64(&ko.processMessageFailures) if msgcount != 0 { t.Errorf("Invalid ending processMessageFailures %d", msgcount) } }
func TcpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests") defer func() { ctrl.Finish() tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() globals := DefaultGlobals() globals.BaseDir = tmpDir pConfig := NewPipelineConfig(globals) pConfig.RegisterDefault("HekaFramingSplitter") c.Specify("TcpOutput", func() { tcpOutput := new(TcpOutput) tcpOutput.SetName("test") config := tcpOutput.ConfigStruct().(*TcpOutputConfig) tcpOutput.Init(config) tickChan := make(chan time.Time) oth := plugins_ts.NewOutputTestHelper(ctrl) oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan).AnyTimes() encoder := new(ProtobufEncoder) encoder.SetPipelineConfig(pConfig) encoder.Init(nil) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.QueueCursor = "queuecursor" outStr := "Write me out to the network" newpack := NewPipelinePack(nil) newpack.Message = msg newpack.Message.SetPayload(outStr) matchBytes, err := proto.Marshal(newpack.Message) c.Assume(err, gs.IsNil) pack.MsgBytes = matchBytes newpack.MsgBytes = matchBytes oth.MockHelper.EXPECT().PipelineConfig().Return(pConfig) c.Specify("doesn't use framing w/o ProtobufEncoder", func() { encoder := new(plugins.PayloadEncoder) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) // We should fail if SetUseFraming is called since we didn't // EXPECT it. }) c.Specify("doesn't use framing if config says not to", func() { useFraming := false config.UseFraming = &useFraming err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) // We should fail if SetUseFraming is called since we didn't // EXPECT it. }) c.Specify("writes out to the network", func() { collectData := func(ch chan string) { ln, err := net.Listen("tcp", "localhost:9125") if err != nil { ch <- err.Error() return } ch <- "ready" conn, err := ln.Accept() if err != nil { ch <- err.Error() return } b := make([]byte, 1000) n, _ := conn.Read(b) ch <- string(b[0:n]) conn.Close() ln.Close() } ch := make(chan string, 1) // don't block on put go collectData(ch) result := <-ch // wait for server err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().SetUseFraming(true) err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor) pack.Message.SetPayload(outStr) msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(0)) err = tcpOutput.ProcessMessage(pack) c.Expect(err, gs.IsNil) result = <-ch msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(1)) c.Expect(result, gs.Equals, string(matchBytes)) tcpOutput.CleanUp() }) c.Specify("far end not initially listening", func() { oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().SetUseFraming(true) err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper) c.Assume(err, gs.IsNil) pack.Message.SetPayload(outStr) msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(0)) err = tcpOutput.ProcessMessage(pack) _, ok := err.(RetryMessageError) c.Expect(ok, gs.IsTrue) msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(0)) // After the message is queued start the collector. However, we // don't have a way guarantee a send attempt has already been made // and that we are actually exercising the retry code. collectData := func(ch chan string) { ln, err := net.Listen("tcp", "localhost:9125") if err != nil { ch <- err.Error() return } ch <- "ready" conn, err := ln.Accept() if err != nil { ch <- err.Error() return } b := make([]byte, 1000) n, _ := conn.Read(b) ch <- string(b[0:n]) conn.Close() ln.Close() } ch := make(chan string, 1) // don't block on put go collectData(ch) result := <-ch oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor) err = tcpOutput.ProcessMessage(pack) c.Expect(err, gs.IsNil) result = <-ch c.Expect(result, gs.Equals, string(matchBytes)) msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(1)) tcpOutput.CleanUp() }) // c.Specify("Overload queue drops messages", func() { // config.QueueFullAction = "drop" // config.QueueMaxBufferSize = uint64(1) // use_framing := false // config.UseFraming = &use_framing // oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) // oth.MockOutputRunner.EXPECT().LogError(QueueIsFull) // err := tcpOutput.Init(config) // c.Expect(err, gs.IsNil) // startOutput() // inChan <- pack // dropcount := atomic.LoadInt64(&tcpOutput.dropMessageCount) // for x := 0; x < 5 && dropcount == 0; x++ { // dropcount = atomic.LoadInt64(&tcpOutput.dropMessageCount) // time.Sleep(time.Duration(100) * time.Millisecond) // } // c.Expect(dropcount, gs.Equals, int64(1)) // close(inChan) // }) // c.Specify("Overload queue shutdowns Heka", func() { // config.QueueFullAction = "shutdown" // config.QueueMaxBufferSize = uint64(1) // use_framing := false // config.UseFraming = &use_framing // oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) // oth.MockOutputRunner.EXPECT().LogError(QueueIsFull) // sigChan := globals.SigChan() // err := tcpOutput.Init(config) // c.Expect(err, gs.IsNil) // startOutput() // inChan <- pack // shutdownSignal := <-sigChan // c.Expect(shutdownSignal, gs.Equals, syscall.SIGINT) // close(inChan) // }) // c.Specify("Overload queue blocks processing until packet is sent", func() { // config.QueueFullAction = "block" // config.QueueMaxBufferSize = uint64(1) // use_framing := false // config.UseFraming = &use_framing // oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes() // oth.MockOutputRunner.EXPECT().LogError(QueueIsFull) // err := tcpOutput.Init(config) // c.Expect(err, gs.IsNil) // startOutput() // inChan <- pack // msgcount := atomic.LoadInt64(&tcpOutput.dropMessageCount) // for x := 0; x < 5 && msgcount == 0; x++ { // msgcount = atomic.LoadInt64(&tcpOutput.dropMessageCount) // time.Sleep(time.Duration(100) * time.Millisecond) // } // c.Expect(atomic.LoadInt64(&tcpOutput.dropMessageCount), gs.Equals, int64(0)) // c.Expect(atomic.LoadInt64(&tcpOutput.processMessageCount), gs.Equals, int64(0)) // close(inChan) // }) }) }
func FileOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() oth := plugins_ts.NewOutputTestHelper(ctrl) var wg sync.WaitGroup inChan := make(chan *PipelinePack, 1) pConfig := NewPipelineConfig(nil) c.Specify("A FileOutput", func() { fileOutput := new(FileOutput) tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano()) tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator), tmpFileName) config := fileOutput.ConfigStruct().(*FileOutputConfig) config.Path = tmpFilePath msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true toString := func(outData interface{}) string { return string(*(outData.(*[]byte))) } c.Specify("correctly formats text output", func() { err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) outData := make([]byte, 0, 20) c.Specify("by default", func() { err = fileOutput.handleMessage(pack, &outData) c.Expect(err, gs.IsNil) c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n") }) c.Specify("w/ a prepended timestamp when specified", func() { fileOutput.prefix_ts = true err = fileOutput.handleMessage(pack, &outData) c.Expect(err, gs.IsNil) // Test will fail if date flips btn handleMessage call and // todayStr calculation... should be extremely rare. todayStr := time.Now().Format("[2006/Jan/02:") strContents := toString(&outData) payload := *msg.Payload c.Expect(strContents, pipeline_ts.StringContains, payload) c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr) }) c.Specify("even when payload is nil", func() { pack.Message.Payload = nil err = fileOutput.handleMessage(pack, &outData) c.Expect(err, gs.IsNil) strContents := toString(&outData) c.Expect(strContents, gs.Equals, "\n") }) c.Specify("payload is nil and with a timestamp", func() { pack.Message.Payload = nil fileOutput.prefix_ts = true err = fileOutput.handleMessage(pack, &outData) c.Expect(err, gs.IsNil) // Test will fail if date flips btn handleMessage call and // todayStr calculation... should be extremely rare. todayStr := time.Now().Format("[2006/Jan/02:") strContents := toString(&outData) c.Expect(strings.HasPrefix(strContents, todayStr), gs.IsTrue) c.Expect(strings.HasSuffix(strContents, " \n"), gs.IsTrue) }) }) c.Specify("correctly formats JSON output", func() { config.Format = "json" err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) outData := make([]byte, 0, 200) c.Specify("when specified", func() { fileOutput.handleMessage(pack, &outData) msgJson, err := json.Marshal(pack.Message) c.Assume(err, gs.IsNil) c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n") }) c.Specify("and with a timestamp", func() { fileOutput.prefix_ts = true fileOutput.handleMessage(pack, &outData) // Test will fail if date flips btn handleMessage call and // todayStr calculation... should be extremely rare. todayStr := time.Now().Format("[2006/Jan/02:") strContents := toString(&outData) msgJson, err := json.Marshal(pack.Message) c.Assume(err, gs.IsNil) c.Expect(strContents, pipeline_ts.StringContains, string(msgJson)+"\n") c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr) }) }) c.Specify("correctly formats protocol buffer stream output", func() { config.Format = "protobufstream" err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) outData := make([]byte, 0, 200) c.Specify("when specified and timestamp ignored", func() { fileOutput.prefix_ts = true err := fileOutput.handleMessage(pack, &outData) c.Expect(err, gs.IsNil) b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue) }) }) c.Specify("processes incoming messages", func() { err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) // Save for comparison. payload := fmt.Sprintf("%s\n", pack.Message.GetPayload()) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) wg.Add(1) go fileOutput.receiver(oth.MockOutputRunner, &wg) inChan <- pack close(inChan) outBatch := <-fileOutput.batchChan wg.Wait() c.Expect(string(outBatch), gs.Equals, payload) }) c.Specify("Init halts if basedirectory is not writable", func() { tmpdir := filepath.Join(os.TempDir(), "tmpdir") err := os.MkdirAll(tmpdir, 0400) c.Assume(err, gs.IsNil) config.Path = tmpdir err = fileOutput.Init(config) c.Assume(err, gs.Not(gs.IsNil)) }) c.Specify("commits to a file", func() { outStr := "Write me out to the log file" outBytes := []byte(outStr) c.Specify("with default settings", func() { err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) // Start committer loop wg.Add(1) go fileOutput.committer(oth.MockOutputRunner, &wg) // Feed and close the batchChan go func() { fileOutput.batchChan <- outBytes _ = <-fileOutput.backChan // clear backChan to prevent blocking close(fileOutput.batchChan) }() wg.Wait() // Wait for the file close operation to happen. //for ; err == nil; _, err = fileOutput.file.Stat() { //} tmpFile, err := os.Open(tmpFilePath) defer tmpFile.Close() c.Assume(err, gs.IsNil) contents, err := ioutil.ReadAll(tmpFile) c.Assume(err, gs.IsNil) c.Expect(string(contents), gs.Equals, outStr) }) c.Specify("with different Perm settings", func() { config.Perm = "600" err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) // Start committer loop wg.Add(1) go fileOutput.committer(oth.MockOutputRunner, &wg) // Feed and close the batchChan go func() { fileOutput.batchChan <- outBytes _ = <-fileOutput.backChan // clear backChan to prevent blocking close(fileOutput.batchChan) }() wg.Wait() // Wait for the file close operation to happen. //for ; err == nil; _, err = fileOutput.file.Stat() { //} tmpFile, err := os.Open(tmpFilePath) defer tmpFile.Close() c.Assume(err, gs.IsNil) fileInfo, err := tmpFile.Stat() c.Assume(err, gs.IsNil) fileMode := fileInfo.Mode() if runtime.GOOS == "windows" { c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-") } else { // 7 consecutive dashes implies no perms for group or other c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------") } }) }) c.Specify("honors folder_perm setting", func() { config.FolderPerm = "750" config.Path = filepath.Join(tmpFilePath, "subfile") err := fileOutput.Init(config) defer os.Remove(config.Path) c.Assume(err, gs.IsNil) fi, err := os.Stat(tmpFilePath) c.Expect(fi.IsDir(), gs.IsTrue) c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750)) }) c.Specify("that starts receiving w/ a flush interval", func() { config.FlushInterval = 100000000 // We'll trigger the timer manually. inChan := make(chan *PipelinePack) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) timerChan := make(chan time.Time) msg2 := pipeline_ts.GetTestMessage() pack2 := NewPipelinePack(pConfig.InputRecycleChan()) pack2.Message = msg2 recvWithConfig := func(config *FileOutputConfig) { err := fileOutput.Init(config) c.Assume(err, gs.IsNil) wg.Add(1) go fileOutput.receiver(oth.MockOutputRunner, &wg) runtime.Gosched() // Yield so we can overwrite the timerChan. fileOutput.timerChan = timerChan } cleanUp := func() { close(inChan) wg.Done() } c.Specify("honors flush interval", func() { recvWithConfig(config) defer cleanUp() inChan <- pack select { case _ = <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") default: } timerChan <- time.Now() select { case _ = <-fileOutput.batchChan: default: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("honors flush interval AND flush count", func() { config.FlushCount = 2 recvWithConfig(config) defer cleanUp() inChan <- pack select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") default: } timerChan <- time.Now() select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") default: } inChan <- pack2 runtime.Gosched() select { case <-fileOutput.batchChan: default: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("honors flush interval OR flush count", func() { config.FlushCount = 2 config.FlushOperator = "OR" recvWithConfig(config) defer cleanUp() inChan <- pack select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") default: } c.Specify("when interval triggers first", func() { timerChan <- time.Now() select { case <-fileOutput.batchChan: default: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("when count triggers first", func() { inChan <- pack2 runtime.Gosched() select { case <-fileOutput.batchChan: default: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) }) }) }) }
func HttpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() var ( reqMethod string reqBody string reqHeader http.Header handleWg sync.WaitGroup runWg sync.WaitGroup delay bool ) handler := new(_test_handler) handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) { defer handleWg.Done() if delay { time.Sleep(time.Duration(2) * time.Millisecond) } // Capture request body for test assertions. p := make([]byte, req.ContentLength) _, err := req.Body.Read(p) c.Expect(err.Error(), gs.Equals, "EOF") reqBody = string(p) // Capture request method and headers for test assertions. reqMethod = req.Method reqHeader = req.Header // Respond with either a response body or a response code. if len(handler.respBody) > 0 { rw.Write([]byte(handler.respBody)) } else { rw.WriteHeader(handler.respCode) } flusher := rw.(http.Flusher) flusher.Flush() } oth := ts.NewOutputTestHelper(ctrl) encoder := new(plugins.PayloadEncoder) encConfig := new(plugins.PayloadEncoderConfig) err := encoder.Init(encConfig) c.Expect(err, gs.IsNil) inChan := make(chan *pipeline.PipelinePack, 1) recycleChan := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(recycleChan) pack.Message = pipeline_ts.GetTestMessage() c.Specify("An HttpOutput", func() { httpOutput := new(HttpOutput) config := httpOutput.ConfigStruct().(*HttpOutputConfig) c.Specify("barfs on bogus URLs", func() { config.Address = "one-two-three-four" err := httpOutput.Init(config) c.Expect(err, gs.Not(gs.IsNil)) }) c.Specify("that is started", func() { server := httptest.NewServer(handler) defer server.Close() runOutput := func() { httpOutput.Run(oth.MockOutputRunner, oth.MockHelper) runWg.Done() } oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) payload := "this is the payload" pack.Message.SetPayload(payload) oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return( []byte(payload), nil) config.Address = server.URL handler.respBody = "Response Body" c.Specify("makes http POST requests by default", func() { err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(reqBody, gs.Equals, payload) c.Expect(reqMethod, gs.Equals, "POST") }) c.Specify("makes http PUT requests", func() { config.Method = "put" err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(reqBody, gs.Equals, payload) c.Expect(reqMethod, gs.Equals, "PUT") }) c.Specify("makes http GET requests", func() { config.Method = "get" err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(reqBody, gs.Equals, "") c.Expect(reqMethod, gs.Equals, "GET") }) c.Specify("correctly passes headers along", func() { config.Headers = http.Header{ "One": []string{"two", "three"}, "Four": []string{"five", "six", "seven"}, } err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(reqBody, gs.Equals, payload) c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"])) c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"])) }) c.Specify("uses http auth when specified", func() { config.Username = "******" config.Password = "******" err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() auth := reqHeader.Get("Authorization") c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue) decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:]) c.Expect(err, gs.IsNil) c.Expect(string(decodedAuth), gs.Equals, "user:pass") }) c.Specify("logs error responses", func() { handler.respBody = "" handler.respCode = 500 err := httpOutput.Init(config) c.Expect(err, gs.IsNil) var errMsg string oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do( func(err error) { errMsg = err.Error() }) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(strings.HasPrefix(errMsg, "HTTP Error code returned: 500"), gs.IsTrue) }) c.Specify("honors http timeout interval", func() { config.HttpTimeout = 1 // 1 millisecond err := httpOutput.Init(config) c.Expect(err, gs.IsNil) var errMsg string oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do( func(err error) { errMsg = err.Error() }) delay = true runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(strings.Contains(errMsg, "use of closed network connection"), gs.IsTrue) }) }) }) }
func SmtpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() oth := plugins_ts.NewOutputTestHelper(ctrl) var wg sync.WaitGroup inChan := make(chan *PipelinePack, 1) pConfig := NewPipelineConfig(nil) encoder := new(plugins.PayloadEncoder) econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig) econfig.AppendNewlines = false encoder.Init(econfig) c.Specify("A SmtpOutput", func() { smtpOutput := new(SmtpOutput) config := smtpOutput.ConfigStruct().(*SmtpOutputConfig) config.SendTo = []string{"root"} msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() inChanCall.Return(inChan) runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes() runnerName.Return("SmtpOutput") oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) encCall := oth.MockOutputRunner.EXPECT().Encode(pack) c.Specify("send email payload message", func() { err := smtpOutput.Init(config) c.Assume(err, gs.IsNil) smtpOutput.sendFunction = testSendMail outStr := "Write me out to the network" pack.Message.SetPayload(outStr) encCall.Return(encoder.Encode(pack)) wg.Add(1) go func() { smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper) wg.Done() }() inChan <- pack close(inChan) wg.Wait() }) }) c.Specify("SmtpOutput Message Body Encoding", func() { smtpOutput := new(SmtpOutput) chars := "123456789012345678901234567890123456789012345678901234567" charsE := "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3" examples := [][]string{ {"Hello", "SGVsbG8="}, {chars, charsE}, {chars + chars, charsE + "\r\n" + charsE}, {chars + chars + "Hello", charsE + "\r\n" + charsE + "\r\n" + "SGVsbG8="}, {"", ""}, {"1", "MQ=="}, } for _, example := range examples { smtpOutput.encodeFullMsg([]byte(example[0])) c.Expect(string(smtpOutput.fullMsg), gs.Equals, example[1]) } }) // // Use this test with a real server // c.Specify("Real SmtpOutput output", func() { // smtpOutput := new(SmtpOutput) // config := smtpOutput.ConfigStruct().(*SmtpOutputConfig) // config.SendTo = []string{"root"} // msg := pipeline_ts.GetTestMessage() // pack := NewPipelinePack(pConfig.InputRecycleChan()) // pack.Message = msg // pack.Decoded = true // inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() // inChanCall.Return(inChan) // runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes() // runnerName.Return("SmtpOutput") // oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) // encCall := oth.MockOutputRunner.EXPECT().Encode(pack) // c.Specify("send a real email essage", func() { // err := smtpOutput.Init(config) // c.Assume(err, gs.IsNil) // outStr := "Write me out to the network" // pack.Message.SetPayload(outStr) // encCall.Return(encoder.Encode(pack)) // go func() { // wg.Add(1) // smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper) // wg.Done() // }() // inChan <- pack // time.Sleep(1000) // allow time for the message output // close(inChan) // wg.Wait() // // manually check the mail // }) // }) }
func FileOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano()) tmpFilePath := filepath.Join(os.TempDir(), tmpFileName) defer func() { ctrl.Finish() os.Remove(tmpFilePath) }() oth := plugins_ts.NewOutputTestHelper(ctrl) var wg sync.WaitGroup inChan := make(chan *PipelinePack, 1) pConfig := NewPipelineConfig(nil) c.Specify("A FileOutput", func() { fileOutput := new(FileOutput) encoder := new(plugins.PayloadEncoder) encoder.Init(encoder.ConfigStruct()) config := fileOutput.ConfigStruct().(*FileOutputConfig) config.Path = tmpFilePath msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.QueueCursor = "queuecursor" errChan := make(chan error, 1) c.Specify("w/ ProtobufEncoder", func() { encoder := new(ProtobufEncoder) encoder.SetPipelineConfig(pConfig) encoder.Init(nil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) c.Specify("uses framing", func() { oth.MockOutputRunner.EXPECT().SetUseFraming(true) err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) wg.Add(1) go func() { err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() close(inChan) wg.Wait() }) c.Specify("but not if config says not to", func() { useFraming := false config.UseFraming = &useFraming err := fileOutput.Init(config) defer os.Remove(tmpFilePath) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) wg.Add(1) go func() { err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() close(inChan) wg.Wait() // We should fail if SetUseFraming is called since we didn't // EXPECT it. }) }) c.Specify("rotates files correctly", func() { config.Path = "%Y-%m-%d" config.RotationInterval = 24 rotateChan := make(chan time.Time) closingChan := make(chan struct{}) err := fileOutput.Init(config) c.Assume(err, gs.IsNil) defer fileOutput.file.Close() fileOutput.rotateChan = rotateChan fileOutput.closing = closingChan fileOutput.startRotateNotifier() committerChan := make(chan struct{}) go func() { fileOutput.committer(oth.MockOutputRunner, errChan) close(committerChan) }() c.Assume(fileOutput.path, gs.Equals, time.Now().Format("2006-01-02")) futureDuration, _ := time.ParseDuration("24h") futureNow := time.Now().Add(futureDuration) rotateChan <- futureNow close(inChan) close(fileOutput.batchChan) <-committerChan c.Assume(fileOutput.path, gs.Equals, futureNow.Format("2006-01-02")) }) c.Specify("processes incoming messages", func() { err := fileOutput.Init(config) c.Assume(err, gs.IsNil) fileOutput.file.Close() // Save for comparison. payload := fmt.Sprintf("%s\n", pack.Message.GetPayload()) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) go fileOutput.receiver(oth.MockOutputRunner, errChan) inChan <- pack close(inChan) outBatch := <-fileOutput.batchChan c.Expect(string(outBatch.data), gs.Equals, payload) c.Expect(outBatch.cursor, gs.Equals, pack.QueueCursor) }) c.Specify("commits to a file", func() { outStr := "Write me out to the log file" outBytes := []byte(outStr) batch := &outBatch{ data: outBytes, cursor: pack.QueueCursor, } oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor) c.Specify("with default settings", func() { err := fileOutput.Init(config) c.Assume(err, gs.IsNil) // Start committer loop. go fileOutput.committer(oth.MockOutputRunner, errChan) // Feed and close the batchChan. go func() { fileOutput.batchChan <- batch _ = <-fileOutput.backChan // clear backChan to prevent blocking. close(fileOutput.batchChan) }() // Wait until we know processing has finished. <-fileOutput.closing tmpFile, err := os.Open(tmpFilePath) defer tmpFile.Close() c.Assume(err, gs.IsNil) contents, err := ioutil.ReadAll(tmpFile) c.Assume(err, gs.IsNil) c.Expect(string(contents), gs.Equals, outStr) }) c.Specify("with different Perm settings", func() { config.Perm = "600" err := fileOutput.Init(config) c.Assume(err, gs.IsNil) // Start committer loop. go fileOutput.committer(oth.MockOutputRunner, errChan) // Feed and close the batchChan. go func() { fileOutput.batchChan <- batch _ = <-fileOutput.backChan // clear backChan to prevent blocking. close(fileOutput.batchChan) }() // Wait until we know processing has finished. <-fileOutput.closing tmpFile, err := os.Open(tmpFilePath) defer tmpFile.Close() c.Assume(err, gs.IsNil) fileInfo, err := tmpFile.Stat() c.Assume(err, gs.IsNil) fileMode := fileInfo.Mode() if runtime.GOOS == "windows" { c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-") } else { // 7 consecutive dashes implies no perms for group or other. c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------") } }) }) if runtime.GOOS != "windows" { if u, err := user.Current(); err != nil && u.Uid != "0" { c.Specify("Init halts if basedirectory is not writable", func() { tmpdir := filepath.Join(os.TempDir(), "tmpdir") err := os.MkdirAll(tmpdir, 0400) c.Assume(err, gs.IsNil) config.Path = filepath.Join(tmpdir, "out.txt") err = fileOutput.Init(config) c.Assume(err, gs.Not(gs.IsNil)) os.RemoveAll(tmpdir) }) } c.Specify("honors folder_perm setting", func() { config.FolderPerm = "750" subdir := filepath.Join(os.TempDir(), "subdir") config.Path = filepath.Join(subdir, "out.txt") err := fileOutput.Init(config) defer os.RemoveAll(subdir) c.Assume(err, gs.IsNil) fi, err := os.Stat(subdir) c.Expect(fi.IsDir(), gs.IsTrue) c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750)) }) } c.Specify("that starts receiving w/ a flush interval", func() { config.FlushInterval = 100000000 // We'll trigger the timer manually. inChan := make(chan *PipelinePack) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) timerChan := make(chan time.Time) msg2 := pipeline_ts.GetTestMessage() msg2.SetPayload("MESSAGE 2") pack2 := NewPipelinePack(pConfig.InputRecycleChan()) pack2.Message = msg2 recvWithConfig := func(config *FileOutputConfig) { err := fileOutput.Init(config) c.Assume(err, gs.IsNil) fileOutput.timerChan = timerChan go fileOutput.receiver(oth.MockOutputRunner, errChan) runtime.Gosched() // Yield so receiver will start. } cleanUp := func() { close(inChan) fileOutput.file.Close() } c.Specify("honors flush interval", func() { oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) recvWithConfig(config) defer cleanUp() inChan <- pack after := time.After(100 * time.Millisecond) select { case _ = <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") case <-after: } timerChan <- time.Now() after = time.After(100 * time.Millisecond) select { case _ = <-fileOutput.batchChan: case <-after: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("honors flush interval AND flush count", func() { oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2)) config.FlushCount = 2 recvWithConfig(config) defer cleanUp() inChan <- pack after := time.After(100 * time.Millisecond) select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") case <-after: } timerChan <- time.Now() after = time.After(100 * time.Millisecond) select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") case <-after: } after = time.After(100 * time.Millisecond) inChan <- pack2 select { case <-fileOutput.batchChan: case <-after: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("honors flush interval OR flush count", func() { oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack)) config.FlushCount = 2 config.FlushOperator = "OR" recvWithConfig(config) defer cleanUp() inChan <- pack after := time.After(100 * time.Millisecond) select { case <-fileOutput.batchChan: c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet") case <-after: } c.Specify("when interval triggers first", func() { timerChan <- time.Now() after = time.After(100 * time.Millisecond) select { case <-fileOutput.batchChan: case <-after: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) c.Specify("when count triggers first", func() { out, err := encoder.Encode(pack2) oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err) inChan <- pack2 after = time.After(100 * time.Millisecond) select { case <-fileOutput.batchChan: case <-after: c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now") } }) }) }) }) }