func StatsdInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) // Specify localhost, but we're not really going to use the network ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, input runner, and stat accumulator ith.MockHelper = NewMockPluginHelper(ctrl) ith.MockInputRunner = NewMockInputRunner(ctrl) mockStatAccum := NewMockStatAccumulator(ctrl) c.Specify("A StatsdInput", func() { statsdInput := StatsdInput{} config := statsdInput.ConfigStruct().(*StatsdInputConfig) config.Address = ith.AddrStr err := statsdInput.Init(config) c.Assume(err, gs.IsNil) realListener := statsdInput.listener c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr) realListener.Close() mockListener := pipeline_ts.NewMockConn(ctrl) statsdInput.listener = mockListener ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil) mockListener.EXPECT().Close() mockListener.EXPECT().SetReadDeadline(gomock.Any()) c.Specify("sends a Stat to the StatAccumulator", func() { statName := "sample.count" statVal := 303 msg := fmt.Sprintf("%s:%d|c\n", statName, statVal) expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)} mockStatAccum.EXPECT().DropStat(expected).Return(true) readCall := mockListener.EXPECT().Read(make([]byte, 512)) readCall.Return(len(msg), nil) readCall.Do(func(msgBytes []byte) { copy(msgBytes, []byte(msg)) statsdInput.Stop() }) var wg sync.WaitGroup wg.Add(1) go func() { err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper) c.Expect(err, gs.IsNil) wg.Done() }() wg.Wait() }) }) }
func TestMatchers(t *testing.T) { type e interface{} type testCase struct { matcher gomock.Matcher yes, no []e } tests := []testCase{ testCase{gomock.Any(), []e{3, nil, "foo"}, nil}, testCase{gomock.Eq(4), []e{4}, []e{3, "blah", nil, int64(4)}}, testCase{gomock.Nil(), []e{nil, (error)(nil), (chan bool)(nil), (*int)(nil)}, []e{"", 0, make(chan bool), errors.New("err"), new(int)}}, testCase{gomock.Not(gomock.Eq(4)), []e{3, "blah", nil, int64(4)}, []e{4}}, } for i, test := range tests { for _, x := range test.yes { if !test.matcher.Matches(x) { t.Errorf(`test %d: "%v %s" should be true.`, i, x, test.matcher) } } for _, x := range test.no { if test.matcher.Matches(x) { t.Errorf(`test %d: "%v %s" should be false.`, i, x, test.matcher) } } } }
func TestSocketListenConfig(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := NewMockStatistician(mockCtrl) mckListenerConfig := NewMockListenerConfig(mockCtrl) app := NewApplication() app.hostname = "example.org" app.SetLogger(mckLogger) app.SetMetrics(mckStat) sh := NewSocketHandler() sh.setApp(app) // Should forward Listen errors. listenErr := errors.New("splines not reticulated") mckListenerConfig.EXPECT().Listen().Return(nil, listenErr) if err := sh.listenWithConfig(mckListenerConfig); err != listenErr { t.Errorf("Wrong error: got %#v; want %#v", err, listenErr) } // Should use the wss:// scheme if UseTLS returns true. ml := newMockListener(netAddr{"test", "[::1]:8080"}) gomock.InOrder( mckListenerConfig.EXPECT().Listen().Return(ml, nil), mckListenerConfig.EXPECT().UseTLS().Return(true), mckListenerConfig.EXPECT().GetMaxConns().Return(1), ) if err := sh.listenWithConfig(mckListenerConfig); err != nil { t.Errorf("Error setting listener: %s", err) } if maxConns := sh.MaxConns(); maxConns != 1 { t.Errorf("Mismatched maximum connection count: got %d; want 1", maxConns) } expectedURL := "wss://example.org:8080" if url := sh.URL(); url != expectedURL { t.Errorf("Mismatched handler URL: got %q; want %q", url, expectedURL) } }
func UdpInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) c.Specify("A UdpInput", func() { udpInput := UdpInput{} config := &UdpInputConfig{} mbytes, _ := proto.Marshal(ith.Msg) header := &message.Header{} header.SetMessageLength(uint32(len(mbytes))) hbytes, _ := proto.Marshal(header) buf := encodeMessage(hbytes, mbytes) bytesChan := make(chan []byte, 1) ith.MockInputRunner.EXPECT().Name().Return("mock_name") ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner) ith.MockSplitterRunner.EXPECT().GetRemainingData().AnyTimes() ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(), nil).AnyTimes() splitCall.Do(func(conn net.Conn, del Deliverer) { recd := make([]byte, 65536) n, _ := conn.Read(recd) recd = recd[:n] bytesChan <- recd }) c.Specify("using a udp address", func() { ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" config.Net = "udp" config.Address = ith.AddrStr err := udpInput.Init(config) c.Assume(err, gs.IsNil) realListener := (udpInput.listener).(*net.UDPConn) c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr) c.Specify("passes the connection to SplitStream", func() { go udpInput.Run(ith.MockInputRunner, ith.MockHelper) conn, err := net.Dial("udp", ith.AddrStr) c.Assume(err, gs.IsNil) _, err = conn.Write(buf) c.Assume(err, gs.IsNil) conn.Close() recd := <-bytesChan c.Expect(string(recd), gs.Equals, string(buf)) udpInput.Stop() }) }) if runtime.GOOS != "windows" { c.Specify("using a unix datagram socket", func() { tmpDir, err := ioutil.TempDir("", "heka-socket") c.Assume(err, gs.IsNil) unixPath := filepath.Join(tmpDir, "unixgram-socket") ith.AddrStr = unixPath config.Net = "unixgram" config.Address = ith.AddrStr err = udpInput.Init(config) c.Assume(err, gs.IsNil) realListener := (udpInput.listener).(*net.UnixConn) c.Expect(realListener.LocalAddr().String(), gs.Equals, unixPath) c.Specify("passes the socket to SplitStream", func() { go udpInput.Run(ith.MockInputRunner, ith.MockHelper) unixAddr, err := net.ResolveUnixAddr("unixgram", unixPath) c.Assume(err, gs.IsNil) conn, err := net.DialUnix("unixgram", nil, unixAddr) c.Assume(err, gs.IsNil) _, err = conn.Write(buf) c.Assume(err, gs.IsNil) conn.Close() recd := <-bytesChan c.Expect(string(recd), gs.Equals, string(buf)) udpInput.Stop() }) }) } }) }
func TestReceivePayloadMessage(t *testing.T) { b1 := sarama.NewMockBroker(t, 1) b2 := sarama.NewMockBroker(t, 2) ctrl := gomock.NewController(t) tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests") if tmpErr != nil { t.Errorf("Unable to create a temporary directory: %s", tmpErr) } defer func() { if err := os.RemoveAll(tmpDir); err != nil { t.Errorf("Cleanup failed: %s", err) } ctrl.Finish() }() topic := "test" mdr := new(sarama.MetadataResponse) mdr.AddBroker(b2.Addr(), b2.BrokerID()) mdr.AddTopicPartition(topic, 0, 2) b1.Returns(mdr) or := new(sarama.OffsetResponse) or.AddTopicPartition(topic, 0, 0) b2.Returns(or) fr := new(sarama.FetchResponse) fr.AddMessage(topic, 0, nil, sarama.ByteEncoder([]byte{0x41, 0x42}), 0) b2.Returns(fr) pConfig := NewPipelineConfig(nil) pConfig.Globals.BaseDir = tmpDir ki := new(KafkaInput) ki.SetName(topic) ki.SetPipelineConfig(pConfig) config := ki.ConfigStruct().(*KafkaInputConfig) config.Addrs = append(config.Addrs, b1.Addr()) config.Topic = topic ith := new(plugins_ts.InputTestHelper) ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) err := ki.Init(config) if err != nil { t.Fatalf("%s", err) } ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) decChan := make(chan func(*PipelinePack), 1) decCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) decCall.Do(func(dec func(pack *PipelinePack)) { decChan <- dec }) bytesChan := make(chan []byte, 1) splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil) splitCall.Do(func(recd []byte, del Deliverer) { bytesChan <- recd }) errChan := make(chan error) go func() { errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper) }() recd := <-bytesChan if string(recd) != "AB" { t.Errorf("Invalid Payload Expected: AB received: %s", string(recd)) } packDec := <-decChan packDec(ith.Pack) if ith.Pack.Message.GetType() != "heka.kafka" { t.Errorf("Invalid Type %s", ith.Pack.Message.GetType()) } // There is a hang on the consumer close with the mock broker // closing the brokers before the consumer works around the issue // and is good enough for this test. b1.Close() b2.Close() ki.Stop() err = <-errChan if err != nil { t.Fatal(err) } filename := filepath.Join(tmpDir, "kafka", "test.test.0.offset.bin") if o, err := readCheckpoint(filename); err != nil { t.Errorf("Could not read the checkpoint file: %s", filename) } else { if o != 1 { t.Errorf("Incorrect offset Expected: 1 Received: %d", o) } } }
func FilePollingInputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) tmpFileName := fmt.Sprintf("filepollinginput-test-%d", time.Now().UnixNano()) tmpFilePath := filepath.Join(os.TempDir(), tmpFileName) defer func() { ctrl.Finish() os.Remove(tmpFilePath) }() pConfig := NewPipelineConfig(nil) var wg sync.WaitGroup errChan := make(chan error, 1) bytesChan := make(chan []byte, 1) tickChan := make(chan time.Time) retPackChan := make(chan *PipelinePack, 2) defer close(retPackChan) c.Specify("A FilePollingInput", func() { input := new(FilePollingInput) ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) config := input.ConfigStruct().(*FilePollingInputConfig) config.FilePath = tmpFilePath startInput := func(msgCount int) { wg.Add(1) go func() { errChan <- input.Run(ith.MockInputRunner, ith.MockHelper) wg.Done() }() } ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) c.Specify("gets updated information when reading a file", func() { err := input.Init(config) c.Assume(err, gs.IsNil) ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(), nil).Return(io.EOF).Times(2) splitCall.Do(func(f *os.File, del Deliverer) { fBytes, err := ioutil.ReadAll(f) if err != nil { fBytes = []byte(err.Error()) } bytesChan <- fBytes }) startInput(2) f, err := os.Create(tmpFilePath) c.Expect(err, gs.IsNil) _, err = f.Write([]byte("test1")) c.Expect(err, gs.IsNil) c.Expect(f.Close(), gs.IsNil) tickChan <- time.Now() msgBytes := <-bytesChan c.Expect(string(msgBytes), gs.Equals, "test1") f, err = os.Create(tmpFilePath) c.Expect(err, gs.IsNil) _, err = f.Write([]byte("test2")) c.Expect(err, gs.IsNil) c.Expect(f.Close(), gs.IsNil) tickChan <- time.Now() msgBytes = <-bytesChan c.Expect(string(msgBytes), gs.Equals, "test2") input.Stop() wg.Wait() c.Expect(<-errChan, gs.IsNil) }) }) }
func TestReceiveProtobufMessage(t *testing.T) { broker := sarama.NewMockBroker(t, 2) ctrl := gomock.NewController(t) tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests") if tmpErr != nil { t.Errorf("Unable to create a temporary directory: %s", tmpErr) } defer func() { if err := os.RemoveAll(tmpDir); err != nil { t.Errorf("Cleanup failed: %s", err) } ctrl.Finish() }() topic := "test" mockFetchResponse := sarama.NewMockFetchResponse(t, 1) mockFetchResponse.SetMessage(topic, 0, 0, sarama.ByteEncoder([]byte{0x41, 0x42})) broker.SetHandlerByMap(map[string]sarama.MockResponse{ "MetadataRequest": sarama.NewMockMetadataResponse(t). SetBroker(broker.Addr(), broker.BrokerID()). SetLeader(topic, 0, broker.BrokerID()), "OffsetRequest": sarama.NewMockOffsetResponse(t). SetOffset(topic, 0, sarama.OffsetOldest, 0). SetOffset(topic, 0, sarama.OffsetNewest, 2), "FetchRequest": mockFetchResponse, }) pConfig := NewPipelineConfig(nil) pConfig.Globals.BaseDir = tmpDir ki := new(KafkaInput) ki.SetName(topic) ki.SetPipelineConfig(pConfig) config := ki.ConfigStruct().(*KafkaInputConfig) config.Addrs = append(config.Addrs, broker.Addr()) config.Topic = topic ith := new(plugins_ts.InputTestHelper) ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) err := ki.Init(config) if err != nil { t.Fatalf("%s", err) } ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true) ith.MockSplitterRunner.EXPECT().Done() bytesChan := make(chan []byte, 1) splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil) splitCall.Do(func(recd []byte, del Deliverer) { bytesChan <- recd }) errChan := make(chan error) go func() { errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper) }() recd := <-bytesChan if string(recd) != "AB" { t.Errorf("Invalid MsgBytes Expected: AB received: %s", string(recd)) } // There is a hang on the consumer close with the mock broker // closing the brokers before the consumer works around the issue // and is good enough for this test. broker.Close() ki.Stop() err = <-errChan if err != nil { t.Fatal(err) } }
func TcpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests") defer func() { ctrl.Finish() tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.Equals, nil) }() globals := DefaultGlobals() globals.BaseDir = tmpDir pConfig := NewPipelineConfig(globals) pConfig.RegisterDefault("HekaFramingSplitter") c.Specify("TcpOutput", func() { tcpOutput := new(TcpOutput) tcpOutput.SetName("test") config := tcpOutput.ConfigStruct().(*TcpOutputConfig) tcpOutput.Init(config) tickChan := make(chan time.Time) oth := plugins_ts.NewOutputTestHelper(ctrl) oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan).AnyTimes() encoder := new(ProtobufEncoder) encoder.SetPipelineConfig(pConfig) encoder.Init(nil) inChan := make(chan *PipelinePack, 1) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true outStr := "Write me out to the network" newpack := NewPipelinePack(nil) newpack.Message = msg newpack.Decoded = true newpack.Message.SetPayload(outStr) matchBytes, err := proto.Marshal(newpack.Message) c.Expect(err, gs.IsNil) inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() inChanCall.Return(inChan) errChan := make(chan error) startOutput := func() { go func() { oth.MockHelper.EXPECT().PipelineConfig().Return(pConfig).AnyTimes() oth.MockOutputRunner.EXPECT().Name().Return("TcpOutput") err := tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() } c.Specify("doesn't use framing w/o ProtobufEncoder", func() { encoder := new(plugins.PayloadEncoder) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) close(inChan) startOutput() err = <-errChan c.Expect(err, gs.IsNil) // We should fail if SetUseFraming is called since we didn't // EXPECT it. }) c.Specify("doesn't use framing if config says not to", func() { useFraming := false config.UseFraming = &useFraming err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) close(inChan) startOutput() err = <-errChan c.Expect(err, gs.IsNil) // We should fail if SetUseFraming is called since we didn't // EXPECT it. }) c.Specify("writes out to the network", func() { collectData := func(ch chan string) { ln, err := net.Listen("tcp", "localhost:9125") if err != nil { ch <- err.Error() return } ch <- "ready" conn, err := ln.Accept() if err != nil { ch <- err.Error() return } b := make([]byte, 1000) n, _ := conn.Read(b) ch <- string(b[0:n]) conn.Close() ln.Close() } ch := make(chan string, 1) // don't block on put go collectData(ch) result := <-ch // wait for server err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().SetUseFraming(true) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes() pack.Message.SetPayload(outStr) startOutput() msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(0)) inChan <- pack result = <-ch msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(1)) c.Expect(result, gs.Equals, string(matchBytes)) close(inChan) err = <-errChan c.Expect(err, gs.IsNil) }) c.Specify("far end not initially listening", func() { oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() err := tcpOutput.Init(config) c.Assume(err, gs.IsNil) pack.Message.SetPayload(outStr) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().SetUseFraming(true) oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes() startOutput() msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount) c.Expect(msgcount, gs.Equals, int64(0)) inChan <- pack for x := 0; x < 5 && msgcount == 0; x++ { msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount) time.Sleep(time.Duration(100) * time.Millisecond) } // After the message is queued start the collector. However, we // don't have a way guarantee a send attempt has already been made // and that we are actually exercising the retry code. collectData := func(ch chan string) { ln, err := net.Listen("tcp", "localhost:9125") if err != nil { ch <- err.Error() return } conn, err := ln.Accept() if err != nil { ch <- err.Error() return } b := make([]byte, 1000) n, _ := conn.Read(b) ch <- string(b[0:n]) conn.Close() ln.Close() } ch := make(chan string, 1) // don't block on put go collectData(ch) result := <-ch c.Expect(result, gs.Equals, string(matchBytes)) close(inChan) err = <-errChan c.Expect(err, gs.IsNil) }) c.Specify("Overload queue drops messages", func() { config.QueueFullAction = "drop" config.QueueMaxBufferSize = uint64(1) use_framing := false config.UseFraming = &use_framing oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().LogError(QueueIsFull) err := tcpOutput.Init(config) c.Expect(err, gs.IsNil) startOutput() inChan <- pack dropcount := atomic.LoadInt64(&tcpOutput.dropMessageCount) for x := 0; x < 5 && dropcount == 0; x++ { dropcount = atomic.LoadInt64(&tcpOutput.dropMessageCount) time.Sleep(time.Duration(100) * time.Millisecond) } c.Expect(dropcount, gs.Equals, int64(1)) close(inChan) }) c.Specify("Overload queue shutdowns Heka", func() { config.QueueFullAction = "shutdown" config.QueueMaxBufferSize = uint64(1) use_framing := false config.UseFraming = &use_framing oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)) oth.MockOutputRunner.EXPECT().LogError(QueueIsFull) sigChan := globals.SigChan() err := tcpOutput.Init(config) c.Expect(err, gs.IsNil) startOutput() inChan <- pack shutdownSignal := <-sigChan c.Expect(shutdownSignal, gs.Equals, syscall.SIGINT) close(inChan) }) c.Specify("Overload queue blocks processing until packet is sent", func() { config.QueueFullAction = "block" config.QueueMaxBufferSize = uint64(1) use_framing := false config.UseFraming = &use_framing oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes() oth.MockOutputRunner.EXPECT().LogError(QueueIsFull) err := tcpOutput.Init(config) c.Expect(err, gs.IsNil) startOutput() inChan <- pack msgcount := atomic.LoadInt64(&tcpOutput.dropMessageCount) for x := 0; x < 5 && msgcount == 0; x++ { msgcount = atomic.LoadInt64(&tcpOutput.dropMessageCount) time.Sleep(time.Duration(100) * time.Millisecond) } c.Expect(atomic.LoadInt64(&tcpOutput.dropMessageCount), gs.Equals, int64(0)) c.Expect(atomic.LoadInt64(&tcpOutput.processMessageCount), gs.Equals, int64(0)) close(inChan) }) }) }
func LogstreamerInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() here, _ := os.Getwd() dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir") tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests") c.Expect(tmpErr, gs.Equals, nil) defer func() { tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.IsNil) }() globals := DefaultGlobals() globals.BaseDir = tmpDir pConfig := NewPipelineConfig(globals) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) // Specify localhost, but we're not really going to use the network. ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // Set up mock helper, runner, and pack supply channel. ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) c.Specify("A LogstreamerInput", func() { lsInput := &LogstreamerInput{pConfig: pConfig} lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig) lsiConfig.LogDirectory = dirPath lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?` lsiConfig.Differentiator = []string{"logfile"} lsiConfig.Priority = []string{"^Seq"} c.Specify("w/ no translation map", func() { err := lsInput.Init(lsiConfig) c.Expect(err, gs.IsNil) c.Expect(len(lsInput.plugins), gs.Equals, 1) // Create pool of packs. numLines := 5 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } c.Specify("reads a log file", func() { // Expect InputRunner calls to get InChan and inject outgoing msgs. ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer) ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return( ith.MockSplitterRunner) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false) ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream( gomock.Any()).Times(numLines) line := "boo hoo foo foo" getRecCall.Return(len(line), []byte(line), nil) getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any()) getRecCall.Return(0, make([]byte, 0), io.EOF) deliverChan := make(chan []byte, 1) deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(), ith.MockDeliverer).Times(numLines) deliverCall.Do(func(record []byte, del Deliverer) { deliverChan <- record }) ith.MockDeliverer.EXPECT().Done() runOutChan := make(chan error, 1) go func() { err = lsInput.Run(ith.MockInputRunner, ith.MockHelper) runOutChan <- err }() dur, _ := time.ParseDuration("5s") timeout := time.After(dur) timed := false for x := 0; x < numLines; x++ { select { case record := <-deliverChan: c.Expect(string(record), gs.Equals, line) case <-timeout: timed = true x += numLines } // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } lsInput.Stop() c.Expect(timed, gs.Equals, false) c.Expect(<-runOutChan, gs.Equals, nil) }) }) c.Specify("with a translation map", func() { lsiConfig.Translation = make(ls.SubmatchTranslationMap) lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap) c.Specify("allows len 1 translation map for 'missing'", func() { lsiConfig.Translation["Seq"]["missing"] = 9999 err := lsInput.Init(lsiConfig) c.Expect(err, gs.IsNil) }) c.Specify("doesn't allow len 1 map for other keys", func() { lsiConfig.Translation["Seq"]["missin"] = 9999 err := lsInput.Init(lsiConfig) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "A translation map with one entry ('Seq') must be specifying a "+ "'missing' key.") }) }) }) }
func TestSocketInvalidOrigin(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := NewMockStatistician(mockCtrl) mckStat.EXPECT().Increment("client.socket.connect").AnyTimes() mckStat.EXPECT().Timer("client.socket.lifespan", gomock.Any()).AnyTimes() mckStat.EXPECT().Increment("client.socket.disconnect").AnyTimes() mckStore := NewMockStore(mockCtrl) mckRouter := NewMockRouter(mockCtrl) testWithOrigin := func(allowedOrigins []string, config *websocket.Config) error { app := NewApplication() app.SetLogger(mckLogger) app.SetMetrics(mckStat) app.SetStore(mckStore) app.SetRouter(mckRouter) sh := NewSocketHandler() defer sh.Close() sh.setApp(app) sh.setOrigins(allowedOrigins) pipe := newPipeListener() if err := sh.listenWithConfig(listenerConfig{listener: pipe}); err != nil { return err } sh.server = newServeWaiter(&http.Server{Handler: sh.ServeMux()}) app.SetSocketHandler(sh) errChan := make(chan error, 1) go sh.Start(errChan) conn, err := dialSocketListener(pipe, config) if err != nil { return err } conn.Close() return nil } location := &url.URL{Scheme: "https", Host: "example.com"} badURL := &url.URL{Scheme: "!@#$", Host: "^&*-"} tests := []struct { name string allowedOrigins []string config *websocket.Config err error }{ { "Should allow all origins if none are specified", nil, &websocket.Config{ Version: websocket.ProtocolVersionHybi13, Location: location, Origin: location, }, nil, }, { "Should match multiple origins", []string{"https://example.com", "https://example.org", "https://example.net"}, &websocket.Config{ Version: websocket.ProtocolVersionHybi13, Location: location, Origin: &url.URL{Scheme: "https", Host: "example.net"}, }, nil, }, { "Should reject mismatched origins", []string{"https://example.com"}, &websocket.Config{ Version: websocket.ProtocolVersionHybi13, Location: location, Origin: &url.URL{Scheme: "http", Host: "example.org"}, }, websocket.ErrBadStatus, // 403 status code. }, { "Should reject malformed origins if some are specified", []string{"https://example.com"}, &websocket.Config{ Version: websocket.ProtocolVersionHybi13, Location: location, Origin: badURL, // Malformed URL. }, websocket.ErrBadStatus, // 403 status code. }, { "Should ignore malformed origins if none are specified", nil, &websocket.Config{ Version: websocket.ProtocolVersionHybi13, Location: location, Origin: badURL, }, nil, }, { "Should allow empty origins if none are specified", nil, &websocket.Config{ Version: websocket.ProtocolVersionHybi13, Location: location, Origin: &url.URL{}, }, nil, }, { "Should reject empty origins if some are specified", []string{"https://example.com"}, &websocket.Config{ Version: websocket.ProtocolVersionHybi13, Location: location, Origin: &url.URL{}, }, websocket.ErrBadStatus, }, } for _, test := range tests { err := testWithOrigin(test.allowedOrigins, test.config) if err != test.err { t.Errorf("On test %s, got %#v; want %#v", test.name, err, test.err) } } }
func SplitterRunnerSpec(c gs.Context) { t := &ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() srConfig := CommonSplitterConfig{} c.Specify("A SplitterRunner w/ HekaFramingSplitter", func() { splitter := &HekaFramingSplitter{} config := splitter.ConfigStruct().(*HekaFramingSplitterConfig) useMsgBytes := true srConfig.UseMsgBytes = &useMsgBytes sr := NewSplitterRunner("HekaFramingSplitter", splitter, srConfig) splitter.SetSplitterRunner(sr) err := splitter.Init(config) c.Assume(err, gs.IsNil) b, err := ioutil.ReadFile(filepath.Join(".", "testsupport", "multi.dat")) c.Assume(err, gs.IsNil) reader := makeMockReader(b) c.Specify("correctly handles data at EOF", func() { count, errCount, bytesRead, foundEOFCount, remainingDataLength, finalRecordLength, eofRecordLength := readRecordsFromStream(sr, reader, true) c.Expect(errCount, gs.Equals, 0) c.Expect(count, gs.Equals, 50) c.Expect(foundEOFCount, gs.Equals, 50) c.Expect(remainingDataLength, gs.Equals, 0) c.Expect(finalRecordLength, gs.Equals, 215) c.Expect(eofRecordLength, gs.Equals, 0) c.Expect(bytesRead, gs.Equals, len(b)) }) c.Specify("correctly splits & unframes a protobuf stream", func() { ir := NewMockInputRunner(ctrl) sr.SetInputRunner(ir) recycleChan := make(chan *PipelinePack, 1) pack := NewPipelinePack(recycleChan) recycleChan <- pack numRecs := 50 ir.EXPECT().InChan().Times(numRecs).Return(recycleChan) delCall := ir.EXPECT().Deliver(pack).Times(numRecs) delCall.Do(func(pack *PipelinePack) { pack.Recycle() }) for err == nil { err = sr.SplitStream(reader, nil) } c.Expect(err, gs.Equals, io.EOF) }) c.Specify("correctly handles appends after EOF", func() { half := len(b) / 2 reader := makeMockReader(b[:half]) totalBytesRead := 0 count, errCount, bytesRead, foundEOFCount, _, finalRecordLength, eofRecordLength := readRecordsFromStream(sr, reader, false) totalBytesRead += bytesRead c.Expect(errCount, gs.Equals, 0) c.Expect(count, gs.Equals, 25) c.Expect(foundEOFCount, gs.Equals, 25) c.Expect(finalRecordLength, gs.Equals, 215) c.Expect(eofRecordLength, gs.Equals, 0) c.Expect(bytesRead <= half, gs.IsTrue) reader.Append(b[half:]) count, errCount, bytesRead, foundEOFCount, remainingDataLength, finalRecordLength, eofRecordLength := readRecordsFromStream(sr, reader, true) totalBytesRead += bytesRead c.Expect(errCount, gs.Equals, 0) c.Expect(count, gs.Equals, 25) c.Expect(foundEOFCount, gs.Equals, 25) c.Expect(remainingDataLength, gs.Equals, 0) c.Expect(finalRecordLength, gs.Equals, 215) c.Expect(eofRecordLength, gs.Equals, 0) c.Expect(totalBytesRead, gs.Equals, len(b)) }) c.Specify("reuse on another stream without GetRemainingData", func() { // Test the case where we reuse the same SplitterRunner on // two different readers, and we do not call GetRemainingData before // using the second reader. half := len(b) / 2 reader1 := makeMockReader(b[:half]) count, errCount, bytesRead, foundEOFCount, _, finalRecordLength, eofRecordLength := readRecordsFromStream(sr, reader1, false) c.Expect(errCount, gs.Equals, 0) c.Expect(count, gs.Equals, 25) c.Expect(foundEOFCount, gs.Equals, 25) c.Expect(finalRecordLength, gs.Equals, 215) c.Expect(eofRecordLength, gs.Equals, 0) leftovers := half - bytesRead c.Expect(leftovers > 0, gs.IsTrue) reader2 := makeMockReader(b) // Don't call GetRemainingData before using sr on a new stream count, errCount, bytesRead, foundEOFCount, remainingDataLength, finalRecordLength, eofRecordLength := readRecordsFromStream(sr, reader2, true) c.Expect(errCount, gs.Equals, 0) c.Expect(count, gs.Equals, 50) c.Expect(foundEOFCount, gs.Equals, 50) c.Expect(remainingDataLength, gs.Equals, 0) c.Expect(finalRecordLength, gs.Equals, 215) c.Expect(eofRecordLength, gs.Equals, 0) // sr misreports the "remaining data" piece from reader1 as being // read from reader2 c.Expect(bytesRead, gs.Equals, len(b)+leftovers) }) c.Specify("reuse on another stream with reset", func() { // Test the case where we reuse the same SplitterRunner on // two different readers, but we call GetRemainingData before using // the second reader. half := len(b) / 2 reader1 := makeMockReader(b[:half]) count, errCount, bytesRead, foundEOFCount, _, finalRecordLength, eofRecordLength := readRecordsFromStream(sr, reader1, false) c.Expect(errCount, gs.Equals, 0) c.Expect(count, gs.Equals, 25) c.Expect(foundEOFCount, gs.Equals, 25) c.Expect(finalRecordLength, gs.Equals, 215) c.Expect(eofRecordLength, gs.Equals, 0) leftovers := half - bytesRead c.Expect(leftovers > 0, gs.IsTrue) reader2 := makeMockReader(b) // Call GetRemainingData before using sr on a new stream sr.GetRemainingData() count, errCount, bytesRead, foundEOFCount, remainingDataLength, finalRecordLength, eofRecordLength := readRecordsFromStream(sr, reader2, true) c.Expect(errCount, gs.Equals, 0) c.Expect(count, gs.Equals, 50) c.Expect(foundEOFCount, gs.Equals, 50) c.Expect(remainingDataLength, gs.Equals, 0) c.Expect(finalRecordLength, gs.Equals, 215) c.Expect(eofRecordLength, gs.Equals, 0) // Now we see the correct number of bytes being read. c.Expect(bytesRead, gs.Equals, len(b)) }) }) c.Specify("A SplitterRunner w/ TokenSplitter", func() { splitter := &TokenSplitter{} config := splitter.ConfigStruct().(*TokenSplitterConfig) c.Specify("sets readPos to 0 when read returns ErrShortBuffer", func() { config.Delimiter = "\t" err := splitter.Init(config) c.Assume(err, gs.IsNil) sr := NewSplitterRunner("TokenSplitter", splitter, srConfig) b := make([]byte, message.MAX_RECORD_SIZE+1) reader := bytes.NewReader(b) var n int var record []byte for err == nil { n, record, err = sr.GetRecordFromStream(reader) } c.Expect(n, gs.Equals, int(message.MAX_RECORD_SIZE)) c.Expect(len(record), gs.Equals, 0) c.Expect(err, gs.Equals, io.ErrShortBuffer) c.Expect(sr.readPos, gs.Equals, 0) c.Expect(sr.scanPos, gs.Equals, 0) }) c.Specify("checks if splitter honors 'deliver_incomplete_final' setting", func() { config.Count = 4 numRecs := 10 err := splitter.Init(config) c.Assume(err, gs.IsNil) packSupply := make(chan *PipelinePack, 1) pack := NewPipelinePack(packSupply) packSupply <- pack ir := NewMockInputRunner(ctrl) // ir.EXPECT().InChan().Return(packSupply).Times(numRecs) // ir.EXPECT().Name().Return("foo").Times(numRecs) ir.EXPECT().InChan().Return(packSupply).AnyTimes() ir.EXPECT().Name().Return("foo").AnyTimes() incompleteFinal := true srConfig.IncompleteFinal = &incompleteFinal sr := NewSplitterRunner("TokenSplitter", splitter, srConfig) sr.ir = ir rExpected := []byte("test1\ntest12\ntest123\npartial\n") buf := bytes.Repeat(rExpected, numRecs) buf = buf[:len(buf)-1] // 40 lines separated by 39 newlines reader := bytes.NewReader(buf) mockDel := NewMockDeliverer(ctrl) delCall := mockDel.EXPECT().Deliver(gomock.Any()).AnyTimes() i := 0 delCall.Do(func(pack *PipelinePack) { i++ if i < numRecs { c.Expect(pack.Message.GetPayload(), gs.Equals, string(rExpected)) } else { c.Expect(pack.Message.GetPayload(), gs.Equals, string(rExpected[:len(rExpected)-1])) } pack.Recycle() }) c.Specify("via SplitStream", func() { for err == nil { err = sr.SplitStream(reader, mockDel) } c.Expect(err, gs.Equals, io.EOF) c.Expect(i, gs.Equals, numRecs) }) c.Specify("via SplitBytes", func() { seekPos, err := sr.SplitBytes(buf, mockDel) c.Assume(err, gs.IsNil) c.Expect(seekPos, gs.Equals, len(buf)) c.Expect(i, gs.Equals, numRecs) }) }) }) }
func AMQPPluginSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) // Our two user/conn waitgroups. ug := new(sync.WaitGroup) cg := new(sync.WaitGroup) // Setup the mock channel. mch := NewMockAMQPChannel(ctrl) // Setup the mock amqpHub with the mock chan return. aqh := NewMockAMQPConnectionHub(ctrl) aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil) errChan := make(chan error, 1) bytesChan := make(chan []byte, 1) c.Specify("An amqp input", func() { // Setup all the mock calls for Init. mch.EXPECT().ExchangeDeclare("", "", false, true, false, false, gomock.Any()).Return(nil) mch.EXPECT().QueueDeclare("", false, true, false, false, gomock.Any()).Return(amqp.Queue{}, nil) mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil) mch.EXPECT().Qos(2, 0, false).Return(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // Set up relevant mocks. ith.MockHelper = NewMockPluginHelper(ctrl) ith.MockInputRunner = NewMockInputRunner(ctrl) ith.MockSplitterRunner = NewMockSplitterRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner) amqpInput := new(AMQPInput) amqpInput.amqpHub = aqh config := amqpInput.ConfigStruct().(*AMQPInputConfig) config.URL = "" config.Exchange = "" config.ExchangeType = "" config.RoutingKey = "test" config.QueueTTL = 300000 err := amqpInput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpInput.ch, gs.Equals, mch) c.Specify("consumes a text message", func() { // Create a channel to send data to the input. Drop a message on // there and close the channel. streamChan := make(chan amqp.Delivery, 1) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "text/plain", Body: []byte("This is a message"), Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Increase the usage since Run decrements it on close. ug.Add(1) splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil) splitCall.Do(func(recd []byte, del Deliverer) { bytesChan <- recd }) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() msgBytes := <-bytesChan c.Expect(string(msgBytes), gs.Equals, "This is a message") close(streamChan) err = <-errChan }) c.Specify("consumes a protobuf encoded message", func() { encoder := client.NewProtobufEncoder(nil) streamChan := make(chan amqp.Delivery, 1) msg := new(message.Message) msg.SetUuid(uuid.NewRandom()) msg.SetTimestamp(time.Now().UnixNano()) msg.SetType("logfile") msg.SetLogger("/a/nice/path") msg.SetSeverity(int32(0)) msg.SetEnvVersion("0.2") msg.SetPid(0) msg.SetPayload("This is a message") msg.SetHostname("TestHost") msgBody := make([]byte, 0, 500) _ = encoder.EncodeMessageStream(msg, &msgBody) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "application/hekad", Body: msgBody, Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Increase the usage since Run decrements it on close. ug.Add(1) splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil) splitCall.Do(func(recd []byte, del Deliverer) { bytesChan <- recd }) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true) go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() msgBytes := <-bytesChan c.Expect(string(msgBytes), gs.Equals, string(msgBody)) close(streamChan) err = <-errChan c.Expect(err, gs.IsNil) }) }) c.Specify("An amqp output", func() { oth := plugins_ts.NewOutputTestHelper(ctrl) pConfig := NewPipelineConfig(nil) amqpOutput := new(AMQPOutput) amqpOutput.amqpHub = aqh config := amqpOutput.ConfigStruct().(*AMQPOutputConfig) config.URL = "" config.Exchange = "" config.ExchangeType = "" config.RoutingKey = "test" closeChan := make(chan *amqp.Error) inChan := make(chan *PipelinePack, 1) mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan) mch.EXPECT().ExchangeDeclare("", "", false, true, false, false, gomock.Any()).Return(nil) // Increase the usage since Run decrements it on close. ug.Add(1) // Expect the close and the InChan calls. aqh.EXPECT().Close("", cg) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true c.Specify("publishes a plain message", func() { encoder := new(plugins.PayloadEncoder) econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig) econfig.AppendNewlines = false encoder.Init(econfig) payloadBytes, err := encoder.Encode(pack) config.Encoder = "PayloadEncoder" config.ContentType = "text/plain" oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil) err = amqpOutput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpOutput.ch, gs.Equals, mch) mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil) inChan <- pack close(inChan) close(closeChan) go func() { err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() ug.Wait() err = <-errChan c.Expect(err, gs.IsNil) }) c.Specify("publishes a serialized message", func() { encoder := new(ProtobufEncoder) encoder.SetPipelineConfig(pConfig) encoder.Init(nil) protoBytes, err := encoder.Encode(pack) c.Expect(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil) err = amqpOutput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpOutput.ch, gs.Equals, mch) mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil) inChan <- pack close(inChan) close(closeChan) go func() { err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() ug.Wait() err = <-errChan c.Expect(err, gs.IsNil) }) }) }
func TestSocketOrigin(t *testing.T) { var err error mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := NewMockStatistician(mockCtrl) mckStore := NewMockStore(mockCtrl) mckRouter := NewMockRouter(mockCtrl) app := NewApplication() app.SetLogger(mckLogger) app.SetMetrics(mckStat) app.SetStore(mckStore) app.SetRouter(mckRouter) sh := NewSocketHandler() defer sh.Close() sh.setApp(app) pipe := newPipeListener() defer pipe.Close() if err := sh.listenWithConfig(listenerConfig{listener: pipe}); err != nil { t.Fatalf("Error setting listener: %s", err) } sh.server = newServeWaiter(&http.Server{Handler: sh.ServeMux()}) app.SetSocketHandler(sh) errChan := make(chan error, 1) go sh.Start(errChan) uaid := "5e1e5984569c4f00bf4bea47754a6403" gomock.InOrder( mckStat.EXPECT().Increment("client.socket.connect"), mckStore.EXPECT().CanStore(0).Return(true), mckRouter.EXPECT().Register(uaid), mckStat.EXPECT().Increment("updates.client.hello"), mckStore.EXPECT().FetchAll(uaid, gomock.Any()).Return(nil, nil, nil), mckStat.EXPECT().Timer("client.flush", gomock.Any()), mckRouter.EXPECT().Unregister(uaid), mckStat.EXPECT().Timer("client.socket.lifespan", gomock.Any()), mckStat.EXPECT().Increment("client.socket.disconnect"), ) origin := &url.URL{Scheme: "https", Host: "example.com"} conn, err := dialSocketListener(pipe, &websocket.Config{ Location: origin, Origin: origin, Version: websocket.ProtocolVersionHybi13, }) if err != nil { t.Fatalf("Error dialing origin: %s", err) } defer conn.Close() err = websocket.JSON.Send(conn, struct { Type string `json:"messageType"` DeviceID string `json:"uaid"` ChannelIDs []string `json:"channelIDs"` }{"hello", uaid, []string{}}) if err != nil { t.Fatalf("Error writing client handshake: %s", err) } reply := new(HelloReply) if err = websocket.JSON.Receive(conn, reply); err != nil { t.Fatalf("Error reading server handshake: %s", err) } if reply.DeviceID != uaid { t.Fatalf("Mismatched device ID: got %q; want %q", reply.DeviceID, uaid) } worker, workerConnected := app.GetWorker(uaid) if !workerConnected { t.Fatalf("Missing worker for device ID %q", uaid) } workerOrigin := worker.Origin() if expectedOrigin := origin.String(); workerOrigin != expectedOrigin { t.Errorf("Mismatched origins: got %q; want %q", workerOrigin, expectedOrigin) } }
func HttpInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'` c.Specify("A HttpInput", func() { httpInput := HttpInput{} ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) runOutputChan := make(chan string, 1) startInput := func() { go func() { err := httpInput.Run(ith.MockInputRunner, ith.MockHelper) var runOutput string if err != nil { runOutput = err.Error() } runOutputChan <- runOutput }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.PackSupply = make(chan *PipelinePack, 1) ith.PackSupply <- ith.Pack config := httpInput.ConfigStruct().(*HttpInputConfig) c.Specify("short circuits packs into the router", func() { config.Url = "http://localhost:9876/" tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) err := httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("with a decoder", func() { decoderName := "TestDecoder" config.DecoderName = decoderName c.Specify("honors time ticker to flush", func() { // Spin up a http server server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876) c.Expect(err, gs.IsNil) go server.Start("/") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9876/" tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("supports configuring HTTP Basic Authentication", func() { // Spin up a http server which expects username "user" and password "password" server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875) c.Expect(err, gs.IsNil) go server.Start("/BasicAuthTest") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9875/BasicAuthTest" config.User = "******" config.Password = "******" tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // we expect a statuscode 200 (i.e. success) pack := <-dRunnerInChan statusCode, ok := pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("supports configuring a different HTTP method", func() { // Spin up a http server which expects requests with method "POST" server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874) c.Expect(err, gs.IsNil) go server.Start("/PostTest") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9874/PostTest" config.Method = "POST" tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // we expect a statuscode 200 (i.e. success) pack := <-dRunnerInChan statusCode, ok := pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("supports configuring HTTP headers", func() { // Spin up a http server which expects requests with method "POST" server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873) c.Expect(err, gs.IsNil) go server.Start("/HeadersTest") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9873/HeadersTest" config.Headers = map[string]string{"Accept": "text/plain"} tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // we expect a statuscode 200 (i.e. success) pack := <-dRunnerInChan statusCode, ok := pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) c.Specify("supports configuring a request body", func() { // Spin up a http server that echoes back the request body server, err := plugins_ts.NewHttpBodyServer("localhost", 9872) c.Expect(err, gs.IsNil) go server.Start("/BodyTest") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9872/BodyTest" config.Method = "POST" config.Body = json_post tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2) ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Stub out the DecoderRunner input channel so that we can // inspect bytes later on dRunnerInChan := make(chan *PipelinePack, 1) mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan) ith.MockInputRunner.EXPECT().Name().Return("HttpInput") ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true) err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() pack := <-dRunnerInChan c.Expect(*pack.Message.Payload, gs.Equals, json_post) // We need for the pipeline to finish up time.Sleep(50 * time.Millisecond) }) ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()) httpInput.Stop() runOutput := <-runOutputChan c.Expect(runOutput, gs.Equals, "") }) }) }
func ProcessInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) c.Specify("A ProcessInput", func() { pInput := ProcessInput{} config := pInput.ConfigStruct().(*ProcessInputConfig) config.Command = make(map[string]cmdConfig) ith.MockHelper.EXPECT().Hostname().Return(pConfig.Hostname()) tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) errChan := make(chan error) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) decChan := make(chan func(*PipelinePack), 1) setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) setDecCall.Do(func(dec func(*PipelinePack)) { decChan <- dec }) bytesChan := make(chan []byte, 1) splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(), ith.MockDeliverer).Return(nil) splitCall.Do(func(r io.Reader, del Deliverer) { bytes, err := ioutil.ReadAll(r) c.Assume(err, gs.IsNil) bytesChan <- bytes }) ith.MockDeliverer.EXPECT().Done() c.Specify("using stdout", func() { ith.MockInputRunner.EXPECT().NewDeliverer("stdout").Return(ith.MockDeliverer) ith.MockInputRunner.EXPECT().NewSplitterRunner("stdout").Return( ith.MockSplitterRunner) c.Specify("reads a message from ProcessInput", func() { pInput.SetName("SimpleTest") // Note that no working directory is explicitly specified. config.Command["0"] = cmdConfig{ Bin: PROCESSINPUT_TEST1_CMD, Args: PROCESSINPUT_TEST1_CMD_ARGS, } err := pInput.Init(config) c.Assume(err, gs.IsNil) go func() { errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper) }() tickChan <- time.Now() actual := <-bytesChan c.Expect(string(actual), gs.Equals, PROCESSINPUT_TEST1_OUTPUT+"\n") dec := <-decChan dec(ith.Pack) fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName") c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout") pInput.Stop() err = <-errChan c.Expect(err, gs.IsNil) }) c.Specify("can pipe multiple commands together", func() { pInput.SetName("PipedCmd") // Note that no working directory is explicitly specified. config.Command["0"] = cmdConfig{ Bin: PROCESSINPUT_PIPE_CMD1, Args: PROCESSINPUT_PIPE_CMD1_ARGS, } config.Command["1"] = cmdConfig{ Bin: PROCESSINPUT_PIPE_CMD2, Args: PROCESSINPUT_PIPE_CMD2_ARGS, } err := pInput.Init(config) c.Assume(err, gs.IsNil) go func() { errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper) }() tickChan <- time.Now() actual := <-bytesChan c.Expect(string(actual), gs.Equals, PROCESSINPUT_PIPE_OUTPUT+"\n") dec := <-decChan dec(ith.Pack) fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName") c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout") pInput.Stop() err = <-errChan c.Expect(err, gs.IsNil) }) }) c.Specify("using stderr", func() { ith.MockInputRunner.EXPECT().NewDeliverer("stderr").Return(ith.MockDeliverer) ith.MockInputRunner.EXPECT().NewSplitterRunner("stderr").Return( ith.MockSplitterRunner) c.Specify("handles bad arguments", func() { pInput.SetName("BadArgs") config.ParseStdout = false config.ParseStderr = true // Note that no working directory is explicitly specified. config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS} err := pInput.Init(config) c.Assume(err, gs.IsNil) expectedErr := fmt.Errorf( "BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]") ith.MockInputRunner.EXPECT().LogError(expectedErr) go func() { errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper) }() tickChan <- time.Now() // Error message differs by platform, but we at least wait // until we get it. <-bytesChan pInput.Stop() err = <-errChan c.Expect(err, gs.IsNil) }) }) }) }
func HttpListenInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) httpListenInput := HttpListenInput{} ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) splitter := &TokenSplitter{} // Not actually used. errChan := make(chan error, 1) startInput := func() { go func() { err := httpListenInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() } config := httpListenInput.ConfigStruct().(*HttpListenInputConfig) config.Address = "127.0.0.1:58325" c.Specify("A HttpListenInput", func() { startedChan := make(chan bool, 1) defer close(startedChan) ts := httptest.NewUnstartedServer(nil) httpListenInput.starterFunc = func(hli *HttpListenInput) error { ts.Start() startedChan <- true return nil } // These EXPECTs imply that every spec below will send exactly one // HTTP request to the input. ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return( ith.MockSplitterRunner) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter) decChan := make(chan func(*PipelinePack), 1) feedDecorator := func(decorator func(*PipelinePack)) { decChan <- decorator } setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) setDecCall.Do(feedDecorator) streamChan := make(chan io.Reader, 1) feedStream := func(r io.Reader) { streamChan <- r } getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream( gomock.Any()).Do(feedStream) bytesChan := make(chan []byte, 1) deliver := func(msgBytes []byte, del Deliverer) { bytesChan <- msgBytes } ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes() c.Specify("Adds query parameters to the message pack as fields", func() { err := httpListenInput.Init(config) c.Assume(err, gs.IsNil) ts.Config = httpListenInput.server getRecCall.Return(0, make([]byte, 0), io.EOF) startInput() <-startedChan resp, err := http.Get(ts.URL + "/?test=Hello%20World") resp.Body.Close() c.Assume(err, gs.IsNil) c.Assume(resp.StatusCode, gs.Equals, 200) packDec := <-decChan packDec(ith.Pack) fieldValue, ok := ith.Pack.Message.GetFieldValue("test") c.Assume(ok, gs.IsTrue) c.Expect(fieldValue, gs.Equals, "Hello World") }) c.Specify("Add custom headers", func() { config.Headers = http.Header{ "One": []string{"two", "three"}, "Four": []string{"five", "six", "seven"}, } err := httpListenInput.Init(config) c.Assume(err, gs.IsNil) ts.Config = httpListenInput.server getRecCall.Return(0, make([]byte, 0), io.EOF) startInput() <-startedChan resp, err := http.Get(ts.URL) c.Assume(err, gs.IsNil) resp.Body.Close() c.Assume(resp.StatusCode, gs.Equals, 200) packDec := <-decChan packDec(ith.Pack) // Verify headers are there eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"]) c.Expect(eq, gs.IsTrue) eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"]) c.Expect(eq, gs.IsTrue) }) c.Specify("Request body is sent as record", func() { err := httpListenInput.Init(config) c.Assume(err, gs.IsNil) ts.Config = httpListenInput.server body := "1+2" getRecCall.Return(0, []byte(body), io.EOF) startInput() <-startedChan deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(), nil) deliverCall.Do(deliver) resp, err := http.Post(ts.URL, "text/plain", strings.NewReader(body)) c.Assume(err, gs.IsNil) resp.Body.Close() c.Assume(resp.StatusCode, gs.Equals, 200) msgBytes := <-bytesChan c.Expect(string(msgBytes), gs.Equals, "1+2") }) c.Specify("Add request headers as fields", func() { config.RequestHeaders = []string{ "X-REQUEST-ID", } err := httpListenInput.Init(config) c.Assume(err, gs.IsNil) ts.Config = httpListenInput.server getRecCall.Return(0, make([]byte, 0), io.EOF) startInput() <-startedChan client := &http.Client{} req, err := http.NewRequest("GET", ts.URL, nil) req.Header.Add("X-REQUEST-ID", "12345") resp, err := client.Do(req) c.Assume(err, gs.IsNil) resp.Body.Close() c.Assume(resp.StatusCode, gs.Equals, 200) packDec := <-decChan packDec(ith.Pack) fieldValue, ok := ith.Pack.Message.GetFieldValue("X-REQUEST-ID") c.Assume(ok, gs.IsTrue) c.Expect(fieldValue, gs.Equals, "12345") }) c.Specify("Add other request properties as fields", func() { err := httpListenInput.Init(config) c.Assume(err, gs.IsNil) ts.Config = httpListenInput.server getRecCall.Return(0, make([]byte, 0), io.EOF) startInput() <-startedChan client := &http.Client{} req, err := http.NewRequest("GET", ts.URL+"/foo/bar?baz=2", nil) c.Assume(err, gs.IsNil) req.Host = "incoming.example.com:8080" resp, err := client.Do(req) resp.Body.Close() c.Assume(err, gs.IsNil) c.Assume(resp.StatusCode, gs.Equals, 200) packDec := <-decChan packDec(ith.Pack) fieldValue, ok := ith.Pack.Message.GetFieldValue("Host") c.Assume(ok, gs.IsTrue) // Host should not include port number. c.Expect(fieldValue, gs.Equals, "incoming.example.com") fieldValue, ok = ith.Pack.Message.GetFieldValue("Path") c.Assume(ok, gs.IsTrue) c.Expect(fieldValue, gs.Equals, "/foo/bar") hostname, err := os.Hostname() c.Assume(err, gs.IsNil) c.Expect(*ith.Pack.Message.Hostname, gs.Equals, hostname) c.Expect(*ith.Pack.Message.EnvVersion, gs.Equals, "1") fieldValue, ok = ith.Pack.Message.GetFieldValue("RemoteAddr") c.Assume(ok, gs.IsTrue) c.Expect(fieldValue != nil, gs.IsTrue) fieldValueStr, ok := fieldValue.(string) c.Assume(ok, gs.IsTrue) c.Expect(len(fieldValueStr) > 0, gs.IsTrue) // Per the `Request` docs, this should be an IP address: // http://golang.org/pkg/net/http/#Request ip := net.ParseIP(fieldValueStr) c.Expect(ip != nil, gs.IsTrue) }) ts.Close() httpListenInput.Stop() err := <-errChan c.Expect(err, gs.IsNil) }) }
func AMQPPluginSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() config := NewPipelineConfig(nil) // Our two user/conn waitgroups ug := new(sync.WaitGroup) cg := new(sync.WaitGroup) // Setup the mock channel mch := NewMockAMQPChannel(ctrl) // Setup the mock amqpHub with the mock chan return aqh := NewMockAMQPConnectionHub(ctrl) aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil) errChan := make(chan error, 1) c.Specify("An amqp input", func() { // Setup all the mock calls for Init mch.EXPECT().ExchangeDeclare("", "", false, true, false, false, gomock.Any()).Return(nil) mch.EXPECT().QueueDeclare("", false, true, false, false, gomock.Any()).Return(amqp.Queue{}, nil) mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil) mch.EXPECT().Qos(2, 0, false).Return(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // set up mock helper, decoder set, and packSupply channel ith.MockHelper = NewMockPluginHelper(ctrl) ith.MockInputRunner = NewMockInputRunner(ctrl) mockDRunner := NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply) amqpInput := new(AMQPInput) amqpInput.amqpHub = aqh config := amqpInput.ConfigStruct().(*AMQPInputConfig) config.URL = "" config.Exchange = "" config.ExchangeType = "" config.RoutingKey = "test" config.QueueTTL = 300000 c.Specify("with a valid setup and no decoder", func() { err := amqpInput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpInput.ch, gs.Equals, mch) c.Specify("consumes a message", func() { // Create a channel to send data to the input // Drop a message on there and close the channel streamChan := make(chan amqp.Delivery, 1) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "text/plain", Body: []byte("This is a message"), Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Expect the injected packet ith.MockInputRunner.EXPECT().Inject(gomock.Any()) // Increase the usage since Run decrements it on close ug.Add(1) ith.PackSupply <- ith.Pack go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() ith.PackSupply <- ith.Pack close(streamChan) err = <-errChan c.Expect(err, gs.IsNil) c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp") c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message") }) }) c.Specify("with a valid setup using a decoder", func() { decoderName := "defaultDecoder" config.Decoder = decoderName err := amqpInput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpInput.ch, gs.Equals, mch) // Mock up our default decoder runner and decoder. ith.MockInputRunner.EXPECT().Name().Return("AMQPInput") decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder") decCall.Return(mockDRunner, true) mockDecoder := NewMockDecoder(ctrl) mockDRunner.EXPECT().Decoder().Return(mockDecoder) c.Specify("consumes a message", func() { packs := []*PipelinePack{ith.Pack} mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil) // Create a channel to send data to the input // Drop a message on there and close the channel streamChan := make(chan amqp.Delivery, 1) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "text/plain", Body: []byte("This is a message"), Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Expect the injected packet ith.MockInputRunner.EXPECT().Inject(gomock.Any()) // Increase the usage since Run decrements it on close ug.Add(1) ith.PackSupply <- ith.Pack go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() ith.PackSupply <- ith.Pack close(streamChan) err = <-errChan c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp") c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message") }) c.Specify("consumes a serialized message", func() { encoder := client.NewProtobufEncoder(nil) streamChan := make(chan amqp.Delivery, 1) msg := new(message.Message) msg.SetUuid(uuid.NewRandom()) msg.SetTimestamp(time.Now().UnixNano()) msg.SetType("logfile") msg.SetLogger("/a/nice/path") msg.SetSeverity(int32(0)) msg.SetEnvVersion("0.2") msg.SetPid(0) msg.SetPayload("This is a message") msg.SetHostname("TestHost") msgBody := make([]byte, 0, 500) _ = encoder.EncodeMessageStream(msg, &msgBody) ack := plugins_ts.NewMockAcknowledger(ctrl) ack.EXPECT().Ack(gomock.Any(), false) streamChan <- amqp.Delivery{ ContentType: "application/hekad", Body: msgBody, Timestamp: time.Now(), Acknowledger: ack, } mch.EXPECT().Consume("", "", false, false, false, false, gomock.Any()).Return(streamChan, nil) // Expect the decoded pack mockDRunner.EXPECT().InChan().Return(ith.DecodeChan) // Increase the usage since Run decrements it on close ug.Add(1) ith.PackSupply <- ith.Pack go func() { err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err }() packRef := <-ith.DecodeChan c.Expect(ith.Pack, gs.Equals, packRef) // Ignore leading 5 bytes of encoded message as thats the header c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:])) ith.PackSupply <- ith.Pack close(streamChan) err = <-errChan c.Expect(err, gs.IsNil) }) }) }) c.Specify("An amqp output", func() { oth := plugins_ts.NewOutputTestHelper(ctrl) pConfig := NewPipelineConfig(nil) amqpOutput := new(AMQPOutput) amqpOutput.amqpHub = aqh config := amqpOutput.ConfigStruct().(*AMQPOutputConfig) config.URL = "" config.Exchange = "" config.ExchangeType = "" config.RoutingKey = "test" closeChan := make(chan *amqp.Error) inChan := make(chan *PipelinePack, 1) mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan) mch.EXPECT().ExchangeDeclare("", "", false, true, false, false, gomock.Any()).Return(nil) // Increase the usage since Run decrements it on close ug.Add(1) // Expect the close and the InChan calls aqh.EXPECT().Close("", cg) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) msg := pipeline_ts.GetTestMessage() pack := NewPipelinePack(pConfig.InputRecycleChan()) pack.Message = msg pack.Decoded = true c.Specify("publishes a plain message", func() { encoder := new(plugins.PayloadEncoder) econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig) econfig.AppendNewlines = false encoder.Init(econfig) payloadBytes, err := encoder.Encode(pack) config.Encoder = "PayloadEncoder" config.ContentType = "text/plain" oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil) err = amqpOutput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpOutput.ch, gs.Equals, mch) mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil) inChan <- pack close(inChan) close(closeChan) go func() { err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() ug.Wait() err = <-errChan c.Expect(err, gs.IsNil) }) c.Specify("publishes a serialized message", func() { encoder := new(ProtobufEncoder) encoder.SetPipelineConfig(pConfig) encoder.Init(nil) protoBytes, err := encoder.Encode(pack) c.Expect(err, gs.IsNil) oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil) err = amqpOutput.Init(config) c.Assume(err, gs.IsNil) c.Expect(amqpOutput.ch, gs.Equals, mch) mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil) inChan <- pack close(inChan) close(closeChan) go func() { err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper) errChan <- err }() ug.Wait() err = <-errChan c.Expect(err, gs.IsNil) }) }) }
func TestEndpointInvalidParams(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := NewMockStatistician(mockCtrl) Convey("Invalid update parameters", t, func() { app := NewApplication() app.SetLogger(mckLogger) app.SetMetrics(mckStat) eh := NewEndpointHandler() eh.setApp(app) eh.setMaxDataLen(512) app.SetEndpointHandler(eh) Convey("Should require PUT requests", func() { resp := httptest.NewRecorder() req := &http.Request{ Method: "POST", Header: http.Header{}, URL: &url.URL{Path: "/update/123"}, } mckStat.EXPECT().Increment("updates.appserver.invalid") eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 405) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, `"Method Not Allowed"`) }) Convey("Should reject negative versions", func() { vals := make(url.Values) vals.Set("version", "-1") resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{}, URL: &url.URL{Path: "/update/123"}, Body: formReader(vals), } mckStat.EXPECT().Increment("updates.appserver.invalid") eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 400) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, `"Invalid Version"`) }) Convey("Should reject invalid versions", func() { vals := make(url.Values) vals.Set("version", "abc") resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{}, URL: &url.URL{Path: "/update/123"}, Body: formReader(vals), } mckStat.EXPECT().Increment("updates.appserver.invalid") eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 400) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, `"Invalid Version"`) }) Convey("Should reject oversized payloads", func() { vals := make(url.Values) vals.Set("data", randomText(eh.maxDataLen+1)) resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{}, URL: &url.URL{Path: "/update/123"}, Body: formReader(vals), } mckStat.EXPECT().Increment("updates.appserver.toolong") eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 413) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, `"Data exceeds max length of 512 bytes"`) }) }) }
func Test_GCMSend(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := &TestMetrics{} mckStat.Init(nil, nil) mckStore := NewMockStore(mockCtrl) mckEndHandler := NewMockHandler(mockCtrl) mckGCMClient := &mockGCMClient{t: t} Convey("GCM Proprietary Ping", t, func() { uaid := "deadbeef00000000000000000000" vers := int64(1) data := "i'm a little teapot" fakeConnect := []byte(`{"regid":"testing"}`) app := NewApplication() app.SetLogger(mckLogger) app.SetMetrics(mckStat) app.SetStore(mckStore) app.SetEndpointHandler(mckEndHandler) mckGCMClient.reset() headers := make(http.Header, 1) headers.Set("Retry-After", "1") mckGCMClient.reply = &http.Response{ Body: respBody("Ok"), StatusCode: 503, Header: headers, } mckStore.EXPECT().FetchPing(uaid).Return(fakeConnect, nil) testGcm := &GCMPing{ logger: app.Logger(), metrics: mckStat, store: mckStore, apiKey: "test_api_key", } conf := testGcm.ConfigStruct() err := testGcm.Init(app, conf) So(err, ShouldBeNil) testGcm.ReplaceClient(mckGCMClient) // For this test, we first respond with a fake 503 message that // requests a "retry after", we follow up with a second message // that returns success. // // Not sure why but when using gomock, calls to Increment are // not being registered, so falling back to older testMetrics // object. ok, err := testGcm.Send(uaid, vers, data) So(err, ShouldBeNil) So(ok, ShouldEqual, true) So(mckStat.Counters["ping.gcm.retry"], ShouldEqual, 1) So(mckStat.Counters["ping.gcm.success"], ShouldEqual, 1) }) }
func TestEndpointDelivery(t *testing.T) { useMockFuncs() defer useStdFuncs() mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := NewMockStatistician(mockCtrl) mckStore := NewMockStore(mockCtrl) mckRouter := NewMockRouter(mockCtrl) mckWorker := NewMockWorker(mockCtrl) Convey("Update delivery", t, func() { app := NewApplication() app.SetLogger(mckLogger) app.SetMetrics(mckStat) app.SetStore(mckStore) app.SetRouter(mckRouter) Convey("Should attempt local delivery if `AlwaysRoute` is disabled", func() { eh := NewEndpointHandler() eh.setApp(app) app.SetEndpointHandler(eh) Convey("Should route updates if the device is not connected", func() { uaid := "f7e9fc483f7344c398701b6fa0e85e4f" chid := "737b7a0d25674be4bb184f015fce02cf" gomock.InOrder( mckStat.EXPECT().Increment("updates.routed.outgoing"), mckRouter.EXPECT().Route(nil, uaid, chid, int64(3), timeNow().UTC(), "", "").Return(true, nil), mckStat.EXPECT().Increment("router.broadcast.hit"), mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()), mckStat.EXPECT().Increment("updates.appserver.received"), ) ok := eh.deliver(nil, uaid, chid, 3, "", "") So(ok, ShouldBeTrue) }) // A routing failure still stores the alert for potential // client delivery. We should only return 404 for absolute // failures (where the endpoint is no longer valid Convey("Should return a 202 if routing fails", func() { resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{HeaderID: {"reqID"}}, URL: &url.URL{Path: "/update/123"}, Body: formReader(url.Values{"version": {"1"}}), } gomock.InOrder( mckStore.EXPECT().KeyToIDs("123").Return("123", "456", nil), mckStat.EXPECT().Increment("updates.appserver.incoming"), mckStore.EXPECT().Update("123", "456", int64(1)).Return(nil), mckStat.EXPECT().Increment("updates.routed.outgoing"), mckRouter.EXPECT().Route(nil, "123", "456", int64(1), gomock.Any(), "reqID", "").Return(false, nil), mckStat.EXPECT().Increment("router.broadcast.miss"), mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()), mckStat.EXPECT().Increment("updates.appserver.rejected"), ) eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 202) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, "{}") }) Convey("Should return a 404 if local delivery fails", func() { uaid := "9e98d6415d8e4fd099ab1bad7178f750" chid := "0eecf572e99f4d508666d8da6c0b15a9" app.AddWorker(uaid, mckWorker) gomock.InOrder( mckWorker.EXPECT().Send(chid, int64(3), "").Return( errors.New("client gone")), mckStat.EXPECT().Increment("updates.appserver.rejected"), ) ok := eh.deliver(nil, uaid, chid, int64(3), "", "") So(ok, ShouldBeFalse) }) Convey("Should return an error if storage is unavailable", func() { resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{}, URL: &url.URL{Path: "/update/123"}, Body: formReader(url.Values{"version": {"2"}}), } updateErr := ErrInvalidChannel gomock.InOrder( mckStore.EXPECT().KeyToIDs("123").Return("123", "456", nil), mckStat.EXPECT().Increment("updates.appserver.incoming"), mckStore.EXPECT().Update("123", "456", int64(2)).Return(updateErr), mckStat.EXPECT().Increment("updates.appserver.error"), ) eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, updateErr.Status()) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, `"Could not update channel version"`) }) }) Convey("Should always route updates if `AlwaysRoute` is enabled", func() { eh := NewEndpointHandler() eh.setApp(app) eh.alwaysRoute = true app.SetEndpointHandler(eh) uaid := "6952a68ee0e7444ebc54f935c4444b13" app.AddWorker(uaid, mckWorker) chid := "b7ede546585f4cc9b95e9340e3406951" version := int64(1) data := "Happy, happy, joy, joy!" Convey("And router delivery fails, local succeeds", func() { gomock.InOrder( mckStat.EXPECT().Increment("updates.routed.outgoing"), mckRouter.EXPECT().Route(nil, uaid, chid, version, gomock.Any(), "", data).Return(false, nil), mckStat.EXPECT().Increment("router.broadcast.miss"), mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()), mckWorker.EXPECT().Send(chid, version, data).Return(nil), mckStat.EXPECT().Increment("updates.appserver.received"), ) ok := eh.deliver(nil, uaid, chid, version, "", data) So(ok, ShouldBeTrue) }) Convey("And router delivery succeeds, local succeeds", func() { gomock.InOrder( mckStat.EXPECT().Increment("updates.routed.outgoing"), mckRouter.EXPECT().Route(nil, uaid, chid, version, gomock.Any(), "", data).Return(true, nil), mckStat.EXPECT().Increment("router.broadcast.hit"), mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()), mckWorker.EXPECT().Send(chid, version, data).Return(nil), mckStat.EXPECT().Increment("updates.appserver.received"), ) ok := eh.deliver(nil, uaid, chid, version, "", data) So(ok, ShouldBeTrue) }) Convey("And router delivery succeeds, local fails", func() { gomock.InOrder( mckStat.EXPECT().Increment("updates.routed.outgoing"), mckRouter.EXPECT().Route(nil, uaid, chid, version, gomock.Any(), "", data).Return(true, nil), mckStat.EXPECT().Increment("router.broadcast.hit"), mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()), mckWorker.EXPECT().Send(chid, version, data).Return( errors.New("client gone")), mckStat.EXPECT().Increment("updates.appserver.received"), ) ok := eh.deliver(nil, uaid, chid, version, "", data) So(ok, ShouldBeTrue) }) Convey("And router/local delivery fails", func() { gomock.InOrder( mckStat.EXPECT().Increment("updates.routed.outgoing"), mckRouter.EXPECT().Route(nil, uaid, chid, version, gomock.Any(), "", data).Return(false, nil), mckStat.EXPECT().Increment("router.broadcast.miss"), mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()), mckWorker.EXPECT().Send(chid, version, data).Return( errors.New("client gone")), mckStat.EXPECT().Increment("updates.appserver.rejected"), ) ok := eh.deliver(nil, uaid, chid, version, "", data) So(ok, ShouldBeFalse) }) }) }) }
func HttpInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'` c.Specify("A HttpInput", func() { httpInput := HttpInput{} ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) runOutputChan := make(chan error, 1) startInput := func() { go func() { runOutputChan <- httpInput.Run(ith.MockInputRunner, ith.MockHelper) }() } ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) // These assume that every sub-spec starts the input. config := httpInput.ConfigStruct().(*HttpInputConfig) tickChan := make(chan time.Time) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) ith.MockHelper.EXPECT().Hostname().Return("hekatests.example.com") // These assume that every sub-spec makes exactly one HTTP request. ith.MockInputRunner.EXPECT().NewSplitterRunner("0").Return(ith.MockSplitterRunner) getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any()) getRecCall.Return(len(json_post), []byte(json_post), io.EOF) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) decChan := make(chan func(*PipelinePack), 1) packDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) packDecCall.Do(func(dec func(*PipelinePack)) { decChan <- dec }) ith.MockSplitterRunner.EXPECT().DeliverRecord([]byte(json_post), nil) ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes() splitter := &TokenSplitter{} // not actually used ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter) c.Specify("honors time ticker to flush", func() { // Spin up a http server. server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876) c.Expect(err, gs.IsNil) go server.Start("/") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9876/" err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() // Getting the decorator means we've made our HTTP request. <-decChan }) c.Specify("supports configuring HTTP Basic Authentication", func() { // Spin up a http server which expects username "user" and password "password" server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875) c.Expect(err, gs.IsNil) go server.Start("/BasicAuthTest") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9875/BasicAuthTest" config.User = "******" config.Password = "******" err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() dec := <-decChan dec(ith.Pack) // we expect a statuscode 200 (i.e. success) statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) }) c.Specify("supports configuring a different HTTP method", func() { // Spin up a http server which expects requests with method "POST" server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874) c.Expect(err, gs.IsNil) go server.Start("/PostTest") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9874/PostTest" config.Method = "POST" err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() dec := <-decChan dec(ith.Pack) // we expect a statuscode 200 (i.e. success) statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) }) c.Specify("supports configuring HTTP headers", func() { // Spin up a http server which expects requests with method "POST" server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873) c.Expect(err, gs.IsNil) go server.Start("/HeadersTest") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9873/HeadersTest" config.Headers = map[string]string{"Accept": "text/plain"} err = httpInput.Init(config) c.Assume(err, gs.IsNil) startInput() tickChan <- time.Now() dec := <-decChan dec(ith.Pack) // we expect a statuscode 200 (i.e. success) statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode") c.Assume(ok, gs.IsTrue) c.Expect(statusCode, gs.Equals, int64(200)) }) c.Specify("supports configuring a request body", func() { // Spin up a http server that echoes back the request body server, err := plugins_ts.NewHttpBodyServer("localhost", 9872) c.Expect(err, gs.IsNil) go server.Start("/BodyTest") time.Sleep(10 * time.Millisecond) config.Url = "http://localhost:9872/BodyTest" config.Method = "POST" config.Body = json_post err = httpInput.Init(config) c.Assume(err, gs.IsNil) respBodyChan := make(chan []byte, 1) getRecCall.Do(func(r io.Reader) { respBody := make([]byte, len(config.Body)) n, err := r.Read(respBody) c.Expect(n, gs.Equals, len(config.Body)) c.Expect(err, gs.Equals, io.EOF) respBodyChan <- respBody }) startInput() tickChan <- time.Now() respBody := <-respBodyChan c.Expect(string(respBody), gs.Equals, json_post) }) httpInput.Stop() runOutput := <-runOutputChan c.Expect(runOutput, gs.IsNil) }) }
func InputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) var wg sync.WaitGroup errChan := make(chan error, 1) tickChan := make(chan time.Time) defer func() { close(tickChan) close(errChan) ctrl.Finish() }() pConfig := NewPipelineConfig(nil) c.Specify("A SandboxInput", func() { input := new(SandboxInput) input.SetPipelineConfig(pConfig) ith := new(plugins_ts.InputTestHelper) ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.Pack = NewPipelinePack(ith.PackSupply) ith.PackSupply <- ith.Pack startInput := func() { wg.Add(1) go func() { errChan <- input.Run(ith.MockInputRunner, ith.MockHelper) wg.Done() }() } c.Specify("test a polling input", func() { tickChan := time.Tick(10 * time.Millisecond) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(2) ith.MockInputRunner.EXPECT().LogError(fmt.Errorf("failure message")) var cnt int ith.MockInputRunner.EXPECT().Inject(gomock.Any()).Do( func(pack *PipelinePack) { switch cnt { case 0: c.Expect(pack.Message.GetPayload(), gs.Equals, "line 1") case 1: c.Expect(pack.Message.GetPayload(), gs.Equals, "line 3") input.Stop() } cnt++ ith.PackSupply <- pack }).Times(2) config := input.ConfigStruct().(*sandbox.SandboxConfig) config.ScriptFilename = "../lua/testsupport/input.lua" err := input.Init(config) c.Assume(err, gs.IsNil) startInput() wg.Wait() c.Expect(<-errChan, gs.IsNil) c.Expect(input.processMessageCount, gs.Equals, int64(2)) c.Expect(input.processMessageFailures, gs.Equals, int64(1)) c.Expect(input.processMessageBytes, gs.Equals, int64(72)) }) c.Specify("run once input", func() { var tickChan <-chan time.Time ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(1) ith.MockInputRunner.EXPECT().LogMessage("single run completed") ith.MockInputRunner.EXPECT().Inject(gomock.Any()).Do( func(pack *PipelinePack) { c.Expect(pack.Message.GetPayload(), gs.Equals, "line 1") ith.PackSupply <- pack }).Times(1) config := input.ConfigStruct().(*sandbox.SandboxConfig) config.ScriptFilename = "../lua/testsupport/input.lua" err := input.Init(config) c.Assume(err, gs.IsNil) startInput() wg.Wait() c.Expect(<-errChan, gs.IsNil) c.Expect(input.processMessageCount, gs.Equals, int64(1)) c.Expect(input.processMessageBytes, gs.Equals, int64(36)) }) c.Specify("exit with error", func() { tickChan := make(chan time.Time) defer close(tickChan) ith.MockInputRunner.EXPECT().Ticker().Return(tickChan) ith.MockInputRunner.EXPECT().LogError(fmt.Errorf("process_message() ../lua/testsupport/input_error.lua:2: boom")) config := input.ConfigStruct().(*sandbox.SandboxConfig) config.ScriptFilename = "../lua/testsupport/input_error.lua" err := input.Init(config) c.Assume(err, gs.IsNil) startInput() wg.Wait() c.Expect(<-errChan, gs.IsNil) c.Expect(input.processMessageCount, gs.Equals, int64(0)) c.Expect(input.processMessageBytes, gs.Equals, int64(0)) }) }) }
func TestEndpointPinger(t *testing.T) { useMockFuncs() defer useStdFuncs() mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := NewMockStatistician(mockCtrl) mckPinger := NewMockPropPinger(mockCtrl) mckStore := NewMockStore(mockCtrl) mckWorker := NewMockWorker(mockCtrl) Convey("Proprietary pings", t, func() { app := NewApplication() app.SetLogger(mckLogger) app.SetMetrics(mckStat) app.SetPropPinger(mckPinger) app.SetStore(mckStore) eh := NewEndpointHandler() eh.setApp(app) eh.setMaxDataLen(4096) app.SetEndpointHandler(eh) Convey("Should return early if the pinger can bypass the WebSocket", func() { uaid := "91357e1a34714cadacb3f13cf47a2736" app.AddWorker(uaid, mckWorker) resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{}, URL: &url.URL{Path: "/update/123"}, Body: nil, } gomock.InOrder( mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil), mckStat.EXPECT().Increment("updates.appserver.incoming"), mckPinger.EXPECT().Send(uaid, int64(1257894000), "").Return(true, nil), mckPinger.EXPECT().CanBypassWebsocket().Return(true), mckStat.EXPECT().Increment("updates.appserver.received"), mckStat.EXPECT().Timer("updates.handled", gomock.Any()), ) eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 200) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, "{}") }) Convey("Should continue if the pinger cannot bypass the WebSocket", func() { uaid := "e3fc2cf1dc44424685010148b076d08b" app.AddWorker(uaid, mckWorker) data := randomText(eh.maxDataLen) vals := make(url.Values) vals.Set("data", data) resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{}, URL: &url.URL{Path: "/update/123"}, Body: formReader(vals), } gomock.InOrder( mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil), mckStat.EXPECT().Increment("updates.appserver.incoming"), mckPinger.EXPECT().Send(uaid, int64(1257894000), data).Return(true, nil), mckPinger.EXPECT().CanBypassWebsocket().Return(false), mckStore.EXPECT().Update(uaid, "456", int64(1257894000)), mckWorker.EXPECT().Send("456", int64(1257894000), data), mckStat.EXPECT().Increment("updates.appserver.received"), mckStat.EXPECT().Timer("updates.handled", gomock.Any()), ) eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 200) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, "{}") }) Convey("Should continue if the pinger fails", func() { uaid := "8f412f5cb2384183bf60f7da26737271" app.AddWorker(uaid, mckWorker) vals := make(url.Values) vals.Set("version", "7") resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{}, URL: &url.URL{Path: "/update/123"}, Body: formReader(vals), } gomock.InOrder( mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil), mckStat.EXPECT().Increment("updates.appserver.incoming"), mckPinger.EXPECT().Send(uaid, int64(7), "").Return( true, errors.New("oops")), mckStore.EXPECT().Update(uaid, "456", int64(7)), mckWorker.EXPECT().Send("456", int64(7), ""), mckStat.EXPECT().Increment("updates.appserver.received"), mckStat.EXPECT().Timer("updates.handled", gomock.Any()), ) eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 200) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, "{}") }) }) }
func TestSocketListenerConfig(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := NewMockStatistician(mockCtrl) tests := []struct { name string hostname string conf listenerConfig url string ok bool }{ { name: "Invalid address, TLS, default hostname", hostname: "example.com", conf: listenerConfig{ listener: newMockListener(netAddr{"test", "!@#$"}), useTLS: true, maxConns: 5, }, url: "wss://example.com", ok: true, }, { name: "IPv6 address, default hostname", hostname: "localhost", conf: listenerConfig{ listener: newMockListener(netAddr{"test", "[::1]:8080"}), useTLS: false, maxConns: 1, }, url: "ws://localhost:8080", ok: true, }, { name: "IPv4 address, TLS, no default hostname", conf: listenerConfig{ listener: newMockListener(netAddr{"test", "127.0.0.1:8090"}), useTLS: true, maxConns: 1, }, url: "wss://127.0.0.1:8090", ok: true, }, } for _, test := range tests { func() { app := NewApplication() if len(test.hostname) > 0 { app.hostname = test.hostname } app.SetLogger(mckLogger) app.SetMetrics(mckStat) sh := NewSocketHandler() defer sh.Close() sh.setApp(app) err := sh.listenWithConfig(test.conf) if err != nil { if test.ok { t.Errorf("On test %s, got listener error: %s", test.name, err) } return } if !test.ok { listener := sh.Listener() t.Errorf("On test %s, got %#v; want listener error", test.name, listener) listener.Close() return } if actual := sh.URL(); actual != test.url { t.Errorf("Mismatched handler URL: got %q; want %q", actual, test.url) } if actual := sh.MaxConns(); actual != test.conf.maxConns { t.Errorf("Mismatched maximum connection count: got %d; want %d", actual, test.conf.maxConns) } }() } }
func TestEndpointResolveKey(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mckStat := NewMockStatistician(mockCtrl) mckStore := NewMockStore(mockCtrl) Convey("Endpoint tokens", t, func() { app := NewApplication() app.SetLogger(mckLogger) app.SetMetrics(mckStat) app.SetStore(mckStore) Convey("Should return a 404 for invalid tokens", func() { app.SetTokenKey("c3v0AlmmxXu_LSfdZY3l3eayLsIwkX48") eh := NewEndpointHandler() eh.setApp(app) app.SetEndpointHandler(eh) resp := httptest.NewRecorder() req := &http.Request{ Method: "PUT", Header: http.Header{}, URL: &url.URL{ Path: "/update/j1bqzFq9WiwFZbqay-y7xVlfSvtO1eY="}, // "123.456" } gomock.InOrder( mckStore.EXPECT().KeyToIDs("123.456").Return("", "", ErrInvalidKey), mckStat.EXPECT().Increment("updates.appserver.invalid"), ) eh.ServeMux().ServeHTTP(resp, req) So(resp.Code, ShouldEqual, 404) body, isJSON := getJSON(resp.HeaderMap, resp.Body) So(isJSON, ShouldBeTrue) So(body.String(), ShouldEqual, `"Invalid Token"`) }) Convey("Should not decode plaintext tokens without a key", func() { var err error app.SetTokenKey("") eh := NewEndpointHandler() eh.setApp(app) app.SetEndpointHandler(eh) _, err = eh.decodePK("") So(err, ShouldNotBeNil) pk, err := eh.decodePK("123.456") So(pk, ShouldEqual, "123.456") }) Convey("Should normalize decoded tokens", func() { app.SetTokenKey("LM1xDImCx0rB46LCnx-3v4-Iyfk1LeKJbx9wuvx_z3U=") eh := NewEndpointHandler() eh.setApp(app) app.SetEndpointHandler(eh) // Hyphenated IDs should be normalized. uaid := "dbda2ba2-004c-491f-9e3d-c5950aee93de" chid := "848cd568-3f2a-4108-9ce4-bd0d928ecad4" // " \t%s.%s\r\n" % (uaid, chid) encodedKey := "qfGSdZzwf20GXiYubmZfIXj11Rx4RGJujFsjSQGdF4LRBhHbB_vt3hdW7cRvL9Fq_t_guMBGkDgebOoa5gRd1GGLN-Cv6h5hkpRTbdju8Tk-hMyC91BP4CEres_8" // decodePK should trim whitespace from encoded keys. mckStore.EXPECT().KeyToIDs( fmt.Sprintf("%s.%s", uaid, chid)).Return(uaid, chid, nil) actualUAID, actualCHID, err := eh.resolvePK(encodedKey) So(err, ShouldBeNil) So(actualUAID, ShouldEqual, uaid) So(actualCHID, ShouldEqual, chid) }) Convey("Should reject invalid tokens", func() { var err error app.SetTokenKey("IhnNwMNbsFWiafTXSgF4Ag==") eh := NewEndpointHandler() eh.setApp(app) app.SetEndpointHandler(eh) invalidKey := "b54QOw2omSWBiEq0IuyfBGxHBIR7AI9YhCMA0lP9" // "_=!@#$%^&*()[]" uaid := "82398a648c834f8b838cb3945eceaf29" chid := "af445ad07e5f46b7a6c858150fc5aa92" validKey := fmt.Sprintf("%s.%s", uaid, chid) encodedKey := "swKSH8P2qprRt5y0J4Wi7ybl-qzFv1j09WPOfuabpEJmVUqwUpxjprXc2R3Yw0ITbqc_Swntw9_EpCgo_XuRTn7Q7opQYoQUgMPhCgT0EGbK" _, _, err = eh.resolvePK(invalidKey[:8]) So(err, ShouldNotBeNil) _, _, err = eh.resolvePK(invalidKey) So(err, ShouldNotBeNil) // Reject plaintext tokens if a key is specified. _, _, err = eh.resolvePK(validKey) So(err, ShouldNotBeNil) mckStore.EXPECT().KeyToIDs(validKey).Return("", "", ErrInvalidKey) _, _, err = eh.resolvePK(encodedKey) So(err, ShouldNotBeNil) mckStore.EXPECT().KeyToIDs(validKey).Return(uaid, chid, nil) actualUAID, actualCHID, err := eh.resolvePK(encodedKey) So(err, ShouldBeNil) So(actualUAID, ShouldEqual, uaid) So(actualCHID, ShouldEqual, chid) }) }) }
func MultiDecoderSpec(c gospec.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) // initializes Globals() c.Specify("A MultiDecoder", func() { subsTOML := `[StartsWithM] type = "PayloadRegexDecoder" match_regex = '^(?P<TheData>m.*)' log_errors = true [StartsWithM.message_fields] StartsWithM = "%TheData%" [StartsWithS] type = "PayloadRegexDecoder" match_regex = '^(?P<TheData>s.*)' log_errors = true [StartsWithS.message_fields] StartsWithS = "%TheData%" [StartsWithM2] type = "PayloadRegexDecoder" match_regex = '^(?P<TheData>m.*)' log_errors = true [StartsWithM2.message_fields] StartsWithM2 = "%TheData%" ` RegisterPlugin("PayloadRegexDecoder", func() interface{} { return &PayloadRegexDecoder{} }) defer delete(AvailablePlugins, "PayloadRegexDecoder") var configFile ConfigFile _, err := toml.Decode(subsTOML, &configFile) c.Assume(err, gs.IsNil) decoder := new(MultiDecoder) decoder.SetName("MyMultiDecoder") decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*MultiDecoderConfig) supply := make(chan *PipelinePack, 1) pack := NewPipelinePack(supply) mSection, ok := configFile["StartsWithM"] c.Assume(ok, gs.IsTrue) mMaker, err := NewPluginMaker("StartsWithM", pConfig, mSection) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["StartsWithM"] = mMaker conf.Subs = []string{"StartsWithM"} errMsg := "All subdecoders failed." dRunner := pipelinemock.NewMockDecoderRunner(ctrl) // An error will be spit out b/c there's no real *dRunner in there; // doesn't impact the tests. dRunner.EXPECT().LogError(gomock.Any()) c.Specify("decodes simple messages", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "matching text" pack.Message.SetPayload(regex_data) _, err = decoder.Decode(pack) c.Assume(err, gs.IsNil) value, ok := pack.Message.GetFieldValue("StartsWithM") c.Assume(ok, gs.IsTrue) c.Expect(value, gs.Equals, regex_data) }) c.Specify("returns an error if all decoders fail", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "non-matching text" pack.Message.SetPayload(regex_data) packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) }) c.Specify("logs subdecoder failures when configured to do so", func() { conf.LogSubErrors = true err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) regex_data := "non-matching text" pack.Message.SetPayload(regex_data) // Expect that we log an error for undecoded message. dRunner.EXPECT().LogError(fmt.Errorf( "Subdecoder 'StartsWithM' decode error: No match: %s", regex_data)).AnyTimes() packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) }) c.Specify("sets subdecoder runner correctly", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) // Call LogError to appease the angry gomock gods. dRunner.LogError(errors.New("foo")) // Now create a real *dRunner, pass it in, make sure a wrapper // gets handed to the subdecoder. dr := NewDecoderRunner(decoder.Name, decoder, 10) decoder.SetDecoderRunner(dr) sub := decoder.Decoders[0] subRunner := sub.(*PayloadRegexDecoder).dRunner c.Expect(subRunner.Name(), gs.Equals, fmt.Sprintf("%s-StartsWithM", decoder.Name)) c.Expect(subRunner.Decoder(), gs.Equals, sub) }) c.Specify("with multiple registered decoders", func() { sSection, ok := configFile["StartsWithS"] c.Assume(ok, gs.IsTrue) sMaker, err := NewPluginMaker("StartsWithS", pConfig, sSection) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["StartsWithS"] = sMaker m2Section, ok := configFile["StartsWithM2"] c.Assume(ok, gs.IsTrue) m2Maker, err := NewPluginMaker("StartsWithM2", pConfig, m2Section) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["StartsWithM2"] = m2Maker conf.Subs = append(conf.Subs, "StartsWithS", "StartsWithM2") // Two more subdecoders means two more LogError calls. dRunner.EXPECT().LogError(gomock.Any()).Times(2) c.Specify("defaults to `first-wins` cascading", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("on a first match condition", func() { pack.Message.SetPayload("match first") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("and a second match condition", func() { pack.Message.SetPayload("second match") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("returning an error if they all fail", func() { pack.Message.SetPayload("won't match") packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) }) c.Specify("and using `all` cascading", func() { conf.CascadeStrategy = "all" err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) c.Specify("matches multiples when appropriate", func() { pack.Message.SetPayload("matches twice") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsTrue) }) c.Specify("matches singles when appropriate", func() { pack.Message.SetPayload("second match") _, err = decoder.Decode(pack) c.Expect(err, gs.IsNil) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsTrue) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) c.Specify("returns an error if they all fail", func() { pack.Message.SetPayload("won't match") packs, err := decoder.Decode(pack) c.Expect(len(packs), gs.Equals, 0) c.Expect(err.Error(), gs.Equals, errMsg) _, ok = pack.Message.GetFieldValue("StartsWithM") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithS") c.Expect(ok, gs.IsFalse) _, ok = pack.Message.GetFieldValue("StartsWithM2") c.Expect(ok, gs.IsFalse) }) }) }) }) c.Specify("A MultiDecoder w/ MultiOutput", func() { subsTOML := `[sub0] type = "MultiOutputDecoder" [sub1] type = "MultiOutputDecoder" [sub2] type = "MultiOutputDecoder" ` RegisterPlugin("MultiOutputDecoder", func() interface{} { return &MultiOutputDecoder{} }) defer delete(AvailablePlugins, "MultiOutputDecoder") var configFile ConfigFile _, err := toml.Decode(subsTOML, &configFile) decoder := new(MultiDecoder) decoder.SetName("MyMultiDecoder") decoder.SetPipelineConfig(pConfig) conf := decoder.ConfigStruct().(*MultiDecoderConfig) conf.CascadeStrategy = "all" supply := make(chan *PipelinePack, 10) pack := NewPipelinePack(supply) sub0Section, ok := configFile["sub0"] c.Assume(ok, gs.IsTrue) sub0Maker, err := NewPluginMaker("sub0", pConfig, sub0Section) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["sub0"] = sub0Maker sub1Section, ok := configFile["sub1"] c.Assume(ok, gs.IsTrue) sub1Maker, err := NewPluginMaker("sub1", pConfig, sub1Section) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["sub1"] = sub1Maker sub2Section, ok := configFile["sub2"] c.Assume(ok, gs.IsTrue) sub2Maker, err := NewPluginMaker("sub2", pConfig, sub2Section) c.Assume(err, gs.IsNil) pConfig.DecoderMakers["sub2"] = sub2Maker conf.Subs = []string{"sub0", "sub1", "sub2"} dRunner := pipelinemock.NewMockDecoderRunner(ctrl) c.Specify("always tries to decode all packs", func() { err := decoder.Init(conf) c.Assume(err, gs.IsNil) decoder.SetDecoderRunner(dRunner) packs, err := decoder.Decode(pack) c.Expect(err, gs.IsNil) c.Expect(len(packs), gs.Equals, 15) }) }) }
func TestLocatorReadyNotify(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() uaid := "fce61180716a40ed8e79bf5ff0ba34bc" mckLogger := NewMockLogger(mockCtrl) mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes() mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() var ( // routerPipes maps fake peer addresses to their respective pipes. Used // by dialRouter to connect to peers. routerPipes = make(map[netAddr]*pipeListener) // contacts is a list of peer URLs for the locator. contacts []string ) // Fake listener for the sender's router, used to test self-routing. sndRouterAddr := netAddr{"tcp", "snd-router.example.com:3000"} sndRouterPipe := newPipeListener() defer sndRouterPipe.Close() routerPipes[sndRouterAddr] = sndRouterPipe contacts = append(contacts, "http://snd-router.example.com:3000") // Fake listener for the receiver's router, used to test routing updates // to different hosts. recvRouterAddr := netAddr{"tcp", "recv-router.example.com:3000"} recvRouterPipe := newPipeListener() defer recvRouterPipe.Close() routerPipes[recvRouterAddr] = recvRouterPipe contacts = append(contacts, "http://recv-router.example.com:3000") // Fake listener for the receiver's WebSocket handler, used to accept a // WebSocket client connection. socketHandlerPipe := newPipeListener() defer socketHandlerPipe.Close() // Fake locator. mckLocator := NewMockLocator(mockCtrl) mckLocator.EXPECT().Contacts(uaid).Return(contacts, nil).Times(2) // Fake dialer to connect to each peer's routing listener. dialRouter := func(network, address string) (net.Conn, error) { if pipe, ok := routerPipes[netAddr{network, address}]; ok { return pipe.Dial(network, address) } return nil, &netErr{temporary: false, timeout: false} } // Configures a fake router for the app. setRouter := func(app *Application, listener net.Listener) { r := NewBroadcastRouter() r.setApp(app) r.setClientOptions(10, 3*time.Second, 3*time.Second) // Defaults. r.setClientTransport(&http.Transport{Dial: dialRouter}) r.listenWithConfig(listenerConfig{listener: listener}) r.maxDataLen = 4096 r.server = newServeWaiter(&http.Server{Handler: r.ServeMux()}) app.SetRouter(r) } // sndApp is the server broadcasting the update. The locator returns the // addresses of the sender and receiver to test self-routing. sndApp := NewApplication() sndApp.SetLogger(mckLogger) sndStat := NewMockStatistician(mockCtrl) sndApp.SetMetrics(sndStat) sndStore := NewMockStore(mockCtrl) sndApp.SetStore(sndStore) sndApp.SetLocator(mckLocator) // Set up a fake router for the sender. setRouter(sndApp, sndRouterPipe) // recvApp is the server receiving the update. recvApp := NewApplication() recvApp.SetLogger(mckLogger) recvStat := NewMockStatistician(mockCtrl) recvApp.SetMetrics(recvStat) recvStore := NewMockStore(mockCtrl) recvApp.SetStore(recvStore) // Wrap the fake locator in a type that implements ReadyNotifier. recvLocator := newMockReadyNotifier(mckLocator) recvApp.SetLocator(recvLocator) // Set up a fake WebSocket handler for the receiver. recvSocketHandler := NewSocketHandler() recvSocketHandler.setApp(recvApp) recvSocketHandler.listenWithConfig(listenerConfig{ listener: socketHandlerPipe}) recvSocketHandler.server = newServeWaiter(&http.Server{Handler: recvSocketHandler.ServeMux()}) recvApp.SetSocketHandler(recvSocketHandler) // Set up a fake router for the receiver. setRouter(recvApp, recvRouterPipe) chid := "2b7c5c27d6224bfeaf1c158c3c57fca3" version := int64(2) data := "I'm a little teapot, short and stout." var wg sync.WaitGroup // Waits for the client to close. wg.Add(1) dialChan := make(chan bool) // Signals when the client connects. timeout := closeAfter(2 * time.Second) go func() { defer wg.Done() origin := &url.URL{Scheme: "ws", Host: "recv-conn.example.com"} ws, err := dialSocketListener(socketHandlerPipe, &websocket.Config{ Location: origin, Origin: origin, Version: websocket.ProtocolVersionHybi13, }) if err != nil { t.Errorf("Error dialing host: %s", err) return } defer ws.Close() err = websocket.JSON.Send(ws, struct { Type string `json:"messageType"` DeviceID string `json:"uaid"` ChannelIDs []string `json:"channelIDs"` }{"hello", uaid, []string{}}) if err != nil { t.Errorf("Error writing handshake request: %s", err) return } helloReply := new(HelloReply) if err = websocket.JSON.Receive(ws, helloReply); err != nil { t.Errorf("Error reading handshake reply: %s", err) return } select { case dialChan <- true: case <-timeout: t.Errorf("Timed out waiting for router") return } flushReply := new(FlushReply) if err = websocket.JSON.Receive(ws, flushReply); err != nil { t.Errorf("Error reading routed update: %s", err) return } ok := false expected := Update{chid, uint64(version), data} for _, update := range flushReply.Updates { if ok = update == expected; ok { break } } if !ok { t.Errorf("Missing update %#v in %#v", expected, flushReply.Updates) return } }() // Start the handlers. errChan := make(chan error, 3) go sndApp.Router().Start(errChan) go recvApp.SocketHandler().Start(errChan) go recvApp.Router().Start(errChan) // First and second routing attempts to self. sndStat.EXPECT().Increment("updates.routed.unknown").Times(2) // Initial routing attempt to peer. recvStat.EXPECT().Increment("updates.routed.unknown") // Client connects to peer. recvStat.EXPECT().Increment("client.socket.connect") recvStore.EXPECT().CanStore(0).Return(true) recvStat.EXPECT().Increment("updates.client.hello") recvStore.EXPECT().FetchAll(uaid, gomock.Any()).Return(nil, nil, nil) recvStat.EXPECT().Timer("client.flush", gomock.Any()) // Second routing attempt to peer. recvStat.EXPECT().Increment("updates.routed.incoming") recvStat.EXPECT().Increment("updates.sent") recvStat.EXPECT().Timer("client.flush", gomock.Any()) recvStat.EXPECT().Increment("updates.routed.received") recvStat.EXPECT().Timer("client.socket.lifespan", gomock.Any()) recvStat.EXPECT().Increment("client.socket.disconnect") // Initial routing attempt should fail; the WebSocket listener shouldn't // accept client connections before the locator is ready. delivered, err := sndApp.Router().Route(nil, uaid, chid, version, timeNow(), "disconnected", data) if err != nil { t.Errorf("Error routing to disconnected client: %s", err) } else if delivered { t.Error("Should not route to disconnected client") } // Signal the locator is ready, then wait for the client to connect. recvLocator.SignalReady() select { case <-dialChan: case <-time.After(5 * time.Second): t.Fatalf("Timed out waiting for the client to connect") } // Routing should succeed once the client is connected. delivered, err = sndApp.Router().Route(nil, uaid, chid, version, timeNow(), "connected", data) if err != nil { t.Errorf("Error routing to connected client: %s", err) } else if !delivered { t.Error("Should route to connected client") } gomock.InOrder( mckLocator.EXPECT().Close(), recvStore.EXPECT().Close(), ) if err := recvApp.Close(); err != nil { t.Errorf("Error closing peer: %s", err) } wg.Wait() gomock.InOrder( mckLocator.EXPECT().Close(), sndStore.EXPECT().Close(), ) if err := sndApp.Close(); err != nil { t.Errorf("Error closing self: %s", err) } // Wait for the handlers to stop. for i := 0; i < 3; i++ { <-errChan } }
func HttpOutputSpec(c gs.Context) { t := new(pipeline_ts.SimpleT) ctrl := gomock.NewController(t) defer ctrl.Finish() var ( reqMethod string reqBody string reqHeader http.Header handleWg sync.WaitGroup runWg sync.WaitGroup delay bool ) handler := new(_test_handler) handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) { defer handleWg.Done() if delay { time.Sleep(time.Duration(2) * time.Millisecond) } // Capture request body for test assertions. p := make([]byte, req.ContentLength) _, err := req.Body.Read(p) c.Expect(err.Error(), gs.Equals, "EOF") reqBody = string(p) // Capture request method and headers for test assertions. reqMethod = req.Method reqHeader = req.Header // Respond with either a response body or a response code. if len(handler.respBody) > 0 { rw.Write([]byte(handler.respBody)) } else { rw.WriteHeader(handler.respCode) } flusher := rw.(http.Flusher) flusher.Flush() } oth := ts.NewOutputTestHelper(ctrl) encoder := new(plugins.PayloadEncoder) encConfig := new(plugins.PayloadEncoderConfig) err := encoder.Init(encConfig) c.Expect(err, gs.IsNil) inChan := make(chan *pipeline.PipelinePack, 1) recycleChan := make(chan *pipeline.PipelinePack, 1) pack := pipeline.NewPipelinePack(recycleChan) pack.Message = pipeline_ts.GetTestMessage() c.Specify("An HttpOutput", func() { httpOutput := new(HttpOutput) config := httpOutput.ConfigStruct().(*HttpOutputConfig) c.Specify("barfs on bogus URLs", func() { config.Address = "one-two-three-four" err := httpOutput.Init(config) c.Expect(err, gs.Not(gs.IsNil)) }) c.Specify("that is started", func() { server := httptest.NewServer(handler) defer server.Close() runOutput := func() { httpOutput.Run(oth.MockOutputRunner, oth.MockHelper) runWg.Done() } oth.MockOutputRunner.EXPECT().Encoder().Return(encoder) oth.MockOutputRunner.EXPECT().InChan().Return(inChan) payload := "this is the payload" pack.Message.SetPayload(payload) oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return( []byte(payload), nil) config.Address = server.URL handler.respBody = "Response Body" c.Specify("makes http POST requests by default", func() { err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(reqBody, gs.Equals, payload) c.Expect(reqMethod, gs.Equals, "POST") }) c.Specify("makes http PUT requests", func() { config.Method = "put" err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(reqBody, gs.Equals, payload) c.Expect(reqMethod, gs.Equals, "PUT") }) c.Specify("makes http GET requests", func() { config.Method = "get" err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(reqBody, gs.Equals, "") c.Expect(reqMethod, gs.Equals, "GET") }) c.Specify("correctly passes headers along", func() { config.Headers = http.Header{ "One": []string{"two", "three"}, "Four": []string{"five", "six", "seven"}, } err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(reqBody, gs.Equals, payload) c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"])) c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"])) }) c.Specify("uses http auth when specified", func() { config.Username = "******" config.Password = "******" err := httpOutput.Init(config) c.Expect(err, gs.IsNil) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() auth := reqHeader.Get("Authorization") c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue) decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:]) c.Expect(err, gs.IsNil) c.Expect(string(decodedAuth), gs.Equals, "user:pass") }) c.Specify("logs error responses", func() { handler.respBody = "" handler.respCode = 500 err := httpOutput.Init(config) c.Expect(err, gs.IsNil) var errMsg string oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do( func(err error) { errMsg = err.Error() }) runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(strings.HasPrefix(errMsg, "HTTP Error code returned: 500"), gs.IsTrue) }) c.Specify("honors http timeout interval", func() { config.HttpTimeout = 1 // 1 millisecond err := httpOutput.Init(config) c.Expect(err, gs.IsNil) var errMsg string oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do( func(err error) { errMsg = err.Error() }) delay = true runWg.Add(1) go runOutput() handleWg.Add(1) inChan <- pack close(inChan) handleWg.Wait() runWg.Wait() c.Expect(strings.Contains(errMsg, "use of closed network connection"), gs.IsTrue) }) }) }) }
func LogstreamerInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() here, _ := os.Getwd() dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir") tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests") c.Expect(tmpErr, gs.Equals, nil) defer func() { tmpErr = os.RemoveAll(tmpDir) c.Expect(tmpErr, gs.IsNil) }() globals := DefaultGlobals() globals.BaseDir = tmpDir config := NewPipelineConfig(globals) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(config.InputRecycleChan()) // Specify localhost, but we're not really going to use the network ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" // set up mock helper, decoder set, and packSupply channel ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) ith.DecodeChan = make(chan *PipelinePack) c.Specify("A LogstreamerInput", func() { lsInput := &LogstreamerInput{pConfig: config} lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig) lsiConfig.LogDirectory = dirPath lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?` lsiConfig.Differentiator = []string{"logfile"} lsiConfig.Priority = []string{"^Seq"} lsiConfig.Decoder = "decoder-name" c.Specify("w/ no translation map", func() { err := lsInput.Init(lsiConfig) c.Expect(err, gs.IsNil) c.Expect(len(lsInput.plugins), gs.Equals, 1) mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl) // Create pool of packs. numLines := 5 // # of lines in the log file we're parsing. packs := make([]*PipelinePack, numLines) ith.PackSupply = make(chan *PipelinePack, numLines) for i := 0; i < numLines; i++ { packs[i] = NewPipelinePack(ith.PackSupply) ith.PackSupply <- packs[i] } c.Specify("reads a log file", func() { // Expect InputRunner calls to get InChan and inject outgoing msgs ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes() ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines) // Expect calls to get decoder and decode each message. Since the // decoding is a no-op, the message payload will be the log file // line, unchanged. pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder, "-"+lsiConfig.Decoder) pbcall.Return(mockDecoderRunner, true) decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines) decodeCall.Return(ith.DecodeChan) runOutChan := make(chan error, 1) go func() { err = lsInput.Run(ith.MockInputRunner, ith.MockHelper) runOutChan <- err }() d, _ := time.ParseDuration("5s") timeout := time.After(d) timed := false for x := 0; x < numLines; x++ { select { case <-ith.DecodeChan: case <-timeout: timed = true x += numLines } // Free up the scheduler while we wait for the log file lines // to be processed. runtime.Gosched() } lsInput.Stop() c.Expect(timed, gs.Equals, false) c.Expect(<-runOutChan, gs.Equals, nil) }) }) c.Specify("with a translation map", func() { lsiConfig.Translation = make(ls.SubmatchTranslationMap) lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap) c.Specify("allows len 1 translation map for 'missing'", func() { lsiConfig.Translation["Seq"]["missing"] = 9999 err := lsInput.Init(lsiConfig) c.Expect(err, gs.IsNil) }) c.Specify("doesn't allow len 1 map for other keys", func() { lsiConfig.Translation["Seq"]["missin"] = 9999 err := lsInput.Init(lsiConfig) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "A translation map with one entry ('Seq') must be specifying a "+ "'missing' key.") }) }) }) }
func TcpInputSpec(c gs.Context) { t := &pipeline_ts.SimpleT{} ctrl := gomock.NewController(t) defer ctrl.Finish() pConfig := NewPipelineConfig(nil) ith := new(plugins_ts.InputTestHelper) ith.Msg = pipeline_ts.GetTestMessage() ith.Pack = NewPipelinePack(pConfig.InputRecycleChan()) ith.AddrStr = "localhost:55565" ith.ResolvedAddrStr = "127.0.0.1:55565" ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl) ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl) ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl) ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl) ith.PackSupply = make(chan *PipelinePack, 1) c.Specify("A TcpInput", func() { tcpInput := &TcpInput{} config := &TcpInputConfig{ Net: "tcp", Address: ith.AddrStr, } bytesChan := make(chan []byte, 1) errChan := make(chan error, 1) var srDoneWG sync.WaitGroup startServer := func() { srDoneWG.Add(1) ith.MockInputRunner.EXPECT().Name().Return("mock_name") ith.MockInputRunner.EXPECT().NewDeliverer(gomock.Any()).Return(ith.MockDeliverer) ith.MockDeliverer.EXPECT().Done() ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return( ith.MockSplitterRunner) ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false) ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any()) ith.MockSplitterRunner.EXPECT().Done().Do(func() { srDoneWG.Done() }) // splitCall gets called twice. The first time it returns nil, the // second time it returns io.EOF. splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(), ith.MockDeliverer).AnyTimes() splitCall.Do(func(conn net.Conn, del Deliverer) { recd, _ := ioutil.ReadAll(conn) bytesChan <- recd splitCall.Return(io.EOF) }) err := tcpInput.Run(ith.MockInputRunner, ith.MockHelper) errChan <- err } c.Specify("not using TLS", func() { err := tcpInput.Init(config) c.Assume(err, gs.IsNil) c.Expect(tcpInput.listener.Addr().String(), gs.Equals, ith.ResolvedAddrStr) c.Specify("accepts connections and passes them to the splitter", func() { go startServer() data := []byte("THIS IS THE DATA") outConn, err := net.Dial("tcp", ith.AddrStr) c.Assume(err, gs.IsNil) _, err = outConn.Write(data) c.Expect(err, gs.IsNil) outConn.Close() recd := <-bytesChan c.Expect(err, gs.IsNil) c.Expect(string(recd), gs.Equals, string(data)) tcpInput.Stop() err = <-errChan c.Expect(err, gs.IsNil) srDoneWG.Wait() }) }) c.Specify("using TLS", func() { config.UseTls = true c.Specify("fails to init w/ missing key or cert file", func() { config.Tls = TlsConfig{} err := tcpInput.Init(config) c.Expect(err, gs.Not(gs.IsNil)) c.Expect(err.Error(), gs.Equals, "TLS config requires both cert_file and key_file value.") }) config.Tls = TlsConfig{ CertFile: "./testsupport/cert.pem", KeyFile: "./testsupport/key.pem", } c.Specify("accepts connections and passes them to the splitter", func() { err := tcpInput.Init(config) c.Expect(err, gs.IsNil) go startServer() data := []byte("This is a test.") clientConfig := new(tls.Config) clientConfig.InsecureSkipVerify = true outConn, err := tls.Dial("tcp", ith.AddrStr, clientConfig) c.Assume(err, gs.IsNil) outConn.SetWriteDeadline(time.Now().Add(time.Duration(10000))) n, err := outConn.Write(data) c.Expect(err, gs.IsNil) c.Expect(n, gs.Equals, len(data)) outConn.Close() recd := <-bytesChan c.Expect(err, gs.IsNil) c.Expect(string(recd), gs.Equals, string(data)) tcpInput.Stop() err = <-errChan c.Expect(err, gs.IsNil) srDoneWG.Wait() }) c.Specify("doesn't accept connections below specified min TLS version", func() { config.Tls.MinVersion = "TLS12" err := tcpInput.Init(config) c.Expect(err, gs.IsNil) go startServer() clientConfig := &tls.Config{ InsecureSkipVerify: true, MaxVersion: tls.VersionTLS11, } conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig) c.Expect(conn, gs.IsNil) c.Expect(err, gs.Not(gs.IsNil)) <-bytesChan tcpInput.Stop() err = <-errChan c.Expect(err, gs.IsNil) srDoneWG.Wait() }) }) }) }