func StatsdInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.PackSupply = make(chan *PipelinePack, 1)
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, input runner, and stat accumulator
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockStatAccum := NewMockStatAccumulator(ctrl)
c.Specify("A StatsdInput", func() {
statsdInput := StatsdInput{}
config := statsdInput.ConfigStruct().(*StatsdInputConfig)
config.Address = ith.AddrStr
err := statsdInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := statsdInput.listener
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockListener := pipeline_ts.NewMockConn(ctrl)
statsdInput.listener = mockListener
ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil)
mockListener.EXPECT().Close()
mockListener.EXPECT().SetReadDeadline(gomock.Any())
c.Specify("sends a Stat to the StatAccumulator", func() {
statName := "sample.count"
statVal := 303
msg := fmt.Sprintf("%s:%d|c\n", statName, statVal)
expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)}
mockStatAccum.EXPECT().DropStat(expected).Return(true)
readCall := mockListener.EXPECT().Read(make([]byte, 512))
readCall.Return(len(msg), nil)
readCall.Do(func(msgBytes []byte) {
copy(msgBytes, []byte(msg))
statsdInput.Stop()
})
var wg sync.WaitGroup
wg.Add(1)
go func() {
err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
wg.Wait()
})
})
}
func TestMatchers(t *testing.T) {
type e interface{}
type testCase struct {
matcher gomock.Matcher
yes, no []e
}
tests := []testCase{
testCase{gomock.Any(), []e{3, nil, "foo"}, nil},
testCase{gomock.Eq(4), []e{4}, []e{3, "blah", nil, int64(4)}},
testCase{gomock.Nil(),
[]e{nil, (error)(nil), (chan bool)(nil), (*int)(nil)},
[]e{"", 0, make(chan bool), errors.New("err"), new(int)}},
testCase{gomock.Not(gomock.Eq(4)), []e{3, "blah", nil, int64(4)}, []e{4}},
}
for i, test := range tests {
for _, x := range test.yes {
if !test.matcher.Matches(x) {
t.Errorf(`test %d: "%v %s" should be true.`, i, x, test.matcher)
}
}
for _, x := range test.no {
if test.matcher.Matches(x) {
t.Errorf(`test %d: "%v %s" should be false.`, i, x, test.matcher)
}
}
}
}
func TestSocketListenConfig(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
gomock.Any(), gomock.Any()).AnyTimes()
mckStat := NewMockStatistician(mockCtrl)
mckListenerConfig := NewMockListenerConfig(mockCtrl)
app := NewApplication()
app.hostname = "example.org"
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
sh := NewSocketHandler()
sh.setApp(app)
// Should forward Listen errors.
listenErr := errors.New("splines not reticulated")
mckListenerConfig.EXPECT().Listen().Return(nil, listenErr)
if err := sh.listenWithConfig(mckListenerConfig); err != listenErr {
t.Errorf("Wrong error: got %#v; want %#v", err, listenErr)
}
// Should use the wss:// scheme if UseTLS returns true.
ml := newMockListener(netAddr{"test", "[::1]:8080"})
gomock.InOrder(
mckListenerConfig.EXPECT().Listen().Return(ml, nil),
mckListenerConfig.EXPECT().UseTLS().Return(true),
mckListenerConfig.EXPECT().GetMaxConns().Return(1),
)
if err := sh.listenWithConfig(mckListenerConfig); err != nil {
t.Errorf("Error setting listener: %s", err)
}
if maxConns := sh.MaxConns(); maxConns != 1 {
t.Errorf("Mismatched maximum connection count: got %d; want 1",
maxConns)
}
expectedURL := "wss://example.org:8080"
if url := sh.URL(); url != expectedURL {
t.Errorf("Mismatched handler URL: got %q; want %q",
url, expectedURL)
}
}
func UdpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
config := &UdpInputConfig{}
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
hbytes, _ := proto.Marshal(header)
buf := encodeMessage(hbytes, mbytes)
bytesChan := make(chan []byte, 1)
ith.MockInputRunner.EXPECT().Name().Return("mock_name")
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().GetRemainingData().AnyTimes()
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
nil).AnyTimes()
splitCall.Do(func(conn net.Conn, del Deliverer) {
recd := make([]byte, 65536)
n, _ := conn.Read(recd)
recd = recd[:n]
bytesChan <- recd
})
c.Specify("using a udp address", func() {
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
config.Net = "udp"
config.Address = ith.AddrStr
err := udpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
c.Specify("passes the connection to SplitStream", func() {
go udpInput.Run(ith.MockInputRunner, ith.MockHelper)
conn, err := net.Dial("udp", ith.AddrStr)
c.Assume(err, gs.IsNil)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
conn.Close()
recd := <-bytesChan
c.Expect(string(recd), gs.Equals, string(buf))
udpInput.Stop()
})
})
if runtime.GOOS != "windows" {
c.Specify("using a unix datagram socket", func() {
tmpDir, err := ioutil.TempDir("", "heka-socket")
c.Assume(err, gs.IsNil)
unixPath := filepath.Join(tmpDir, "unixgram-socket")
ith.AddrStr = unixPath
config.Net = "unixgram"
config.Address = ith.AddrStr
err = udpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UnixConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, unixPath)
c.Specify("passes the socket to SplitStream", func() {
go udpInput.Run(ith.MockInputRunner, ith.MockHelper)
unixAddr, err := net.ResolveUnixAddr("unixgram", unixPath)
c.Assume(err, gs.IsNil)
conn, err := net.DialUnix("unixgram", nil, unixAddr)
c.Assume(err, gs.IsNil)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
conn.Close()
recd := <-bytesChan
c.Expect(string(recd), gs.Equals, string(buf))
udpInput.Stop()
})
})
}
})
}
func TestReceivePayloadMessage(t *testing.T) {
b1 := sarama.NewMockBroker(t, 1)
b2 := sarama.NewMockBroker(t, 2)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests")
if tmpErr != nil {
t.Errorf("Unable to create a temporary directory: %s", tmpErr)
}
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Errorf("Cleanup failed: %s", err)
}
ctrl.Finish()
}()
topic := "test"
mdr := new(sarama.MetadataResponse)
mdr.AddBroker(b2.Addr(), b2.BrokerID())
mdr.AddTopicPartition(topic, 0, 2)
b1.Returns(mdr)
or := new(sarama.OffsetResponse)
or.AddTopicPartition(topic, 0, 0)
b2.Returns(or)
fr := new(sarama.FetchResponse)
fr.AddMessage(topic, 0, nil, sarama.ByteEncoder([]byte{0x41, 0x42}), 0)
b2.Returns(fr)
pConfig := NewPipelineConfig(nil)
pConfig.Globals.BaseDir = tmpDir
ki := new(KafkaInput)
ki.SetName(topic)
ki.SetPipelineConfig(pConfig)
config := ki.ConfigStruct().(*KafkaInputConfig)
config.Addrs = append(config.Addrs, b1.Addr())
config.Topic = topic
ith := new(plugins_ts.InputTestHelper)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
err := ki.Init(config)
if err != nil {
t.Fatalf("%s", err)
}
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
decChan := make(chan func(*PipelinePack), 1)
decCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
decCall.Do(func(dec func(pack *PipelinePack)) {
decChan <- dec
})
bytesChan := make(chan []byte, 1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
errChan := make(chan error)
go func() {
errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper)
}()
recd := <-bytesChan
if string(recd) != "AB" {
t.Errorf("Invalid Payload Expected: AB received: %s", string(recd))
}
packDec := <-decChan
packDec(ith.Pack)
if ith.Pack.Message.GetType() != "heka.kafka" {
t.Errorf("Invalid Type %s", ith.Pack.Message.GetType())
}
// There is a hang on the consumer close with the mock broker
// closing the brokers before the consumer works around the issue
// and is good enough for this test.
b1.Close()
b2.Close()
ki.Stop()
err = <-errChan
if err != nil {
t.Fatal(err)
}
filename := filepath.Join(tmpDir, "kafka", "test.test.0.offset.bin")
if o, err := readCheckpoint(filename); err != nil {
t.Errorf("Could not read the checkpoint file: %s", filename)
} else {
if o != 1 {
t.Errorf("Incorrect offset Expected: 1 Received: %d", o)
}
}
}
func FilePollingInputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("filepollinginput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
pConfig := NewPipelineConfig(nil)
var wg sync.WaitGroup
errChan := make(chan error, 1)
bytesChan := make(chan []byte, 1)
tickChan := make(chan time.Time)
retPackChan := make(chan *PipelinePack, 2)
defer close(retPackChan)
c.Specify("A FilePollingInput", func() {
input := new(FilePollingInput)
ith := new(plugins_ts.InputTestHelper)
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
config := input.ConfigStruct().(*FilePollingInputConfig)
config.FilePath = tmpFilePath
startInput := func(msgCount int) {
wg.Add(1)
go func() {
errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
wg.Done()
}()
}
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
c.Specify("gets updated information when reading a file", func() {
err := input.Init(config)
c.Assume(err, gs.IsNil)
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
nil).Return(io.EOF).Times(2)
splitCall.Do(func(f *os.File, del Deliverer) {
fBytes, err := ioutil.ReadAll(f)
if err != nil {
fBytes = []byte(err.Error())
}
bytesChan <- fBytes
})
startInput(2)
f, err := os.Create(tmpFilePath)
c.Expect(err, gs.IsNil)
_, err = f.Write([]byte("test1"))
c.Expect(err, gs.IsNil)
c.Expect(f.Close(), gs.IsNil)
tickChan <- time.Now()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "test1")
f, err = os.Create(tmpFilePath)
c.Expect(err, gs.IsNil)
_, err = f.Write([]byte("test2"))
c.Expect(err, gs.IsNil)
c.Expect(f.Close(), gs.IsNil)
tickChan <- time.Now()
msgBytes = <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "test2")
input.Stop()
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
})
})
}
func TestReceiveProtobufMessage(t *testing.T) {
broker := sarama.NewMockBroker(t, 2)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests")
if tmpErr != nil {
t.Errorf("Unable to create a temporary directory: %s", tmpErr)
}
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Errorf("Cleanup failed: %s", err)
}
ctrl.Finish()
}()
topic := "test"
mockFetchResponse := sarama.NewMockFetchResponse(t, 1)
mockFetchResponse.SetMessage(topic, 0, 0, sarama.ByteEncoder([]byte{0x41, 0x42}))
broker.SetHandlerByMap(map[string]sarama.MockResponse{
"MetadataRequest": sarama.NewMockMetadataResponse(t).
SetBroker(broker.Addr(), broker.BrokerID()).
SetLeader(topic, 0, broker.BrokerID()),
"OffsetRequest": sarama.NewMockOffsetResponse(t).
SetOffset(topic, 0, sarama.OffsetOldest, 0).
SetOffset(topic, 0, sarama.OffsetNewest, 2),
"FetchRequest": mockFetchResponse,
})
pConfig := NewPipelineConfig(nil)
pConfig.Globals.BaseDir = tmpDir
ki := new(KafkaInput)
ki.SetName(topic)
ki.SetPipelineConfig(pConfig)
config := ki.ConfigStruct().(*KafkaInputConfig)
config.Addrs = append(config.Addrs, broker.Addr())
config.Topic = topic
ith := new(plugins_ts.InputTestHelper)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
err := ki.Init(config)
if err != nil {
t.Fatalf("%s", err)
}
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
ith.MockSplitterRunner.EXPECT().Done()
bytesChan := make(chan []byte, 1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
errChan := make(chan error)
go func() {
errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper)
}()
recd := <-bytesChan
if string(recd) != "AB" {
t.Errorf("Invalid MsgBytes Expected: AB received: %s", string(recd))
}
// There is a hang on the consumer close with the mock broker
// closing the brokers before the consumer works around the issue
// and is good enough for this test.
broker.Close()
ki.Stop()
err = <-errChan
if err != nil {
t.Fatal(err)
}
}
func TcpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests")
defer func() {
ctrl.Finish()
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
pConfig := NewPipelineConfig(globals)
pConfig.RegisterDefault("HekaFramingSplitter")
c.Specify("TcpOutput", func() {
tcpOutput := new(TcpOutput)
tcpOutput.SetName("test")
config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
tcpOutput.Init(config)
tickChan := make(chan time.Time)
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan).AnyTimes()
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
inChan := make(chan *PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
outStr := "Write me out to the network"
newpack := NewPipelinePack(nil)
newpack.Message = msg
newpack.Decoded = true
newpack.Message.SetPayload(outStr)
matchBytes, err := proto.Marshal(newpack.Message)
c.Expect(err, gs.IsNil)
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
errChan := make(chan error)
startOutput := func() {
go func() {
oth.MockHelper.EXPECT().PipelineConfig().Return(pConfig).AnyTimes()
oth.MockOutputRunner.EXPECT().Name().Return("TcpOutput")
err := tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
}
c.Specify("doesn't use framing w/o ProtobufEncoder", func() {
encoder := new(plugins.PayloadEncoder)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
startOutput()
err = <-errChan
c.Expect(err, gs.IsNil)
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
c.Specify("doesn't use framing if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
close(inChan)
startOutput()
err = <-errChan
c.Expect(err, gs.IsNil)
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
c.Specify("writes out to the network", func() {
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch // wait for server
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()
pack.Message.SetPayload(outStr)
startOutput()
msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
inChan <- pack
result = <-ch
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(1))
c.Expect(result, gs.Equals, string(matchBytes))
close(inChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("far end not initially listening", func() {
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
pack.Message.SetPayload(outStr)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()
startOutput()
msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
inChan <- pack
for x := 0; x < 5 && msgcount == 0; x++ {
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
time.Sleep(time.Duration(100) * time.Millisecond)
}
// After the message is queued start the collector. However, we
// don't have a way guarantee a send attempt has already been made
// and that we are actually exercising the retry code.
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch
c.Expect(result, gs.Equals, string(matchBytes))
close(inChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("Overload queue drops messages", func() {
config.QueueFullAction = "drop"
config.QueueMaxBufferSize = uint64(1)
use_framing := false
config.UseFraming = &use_framing
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)
err := tcpOutput.Init(config)
c.Expect(err, gs.IsNil)
startOutput()
inChan <- pack
dropcount := atomic.LoadInt64(&tcpOutput.dropMessageCount)
for x := 0; x < 5 && dropcount == 0; x++ {
dropcount = atomic.LoadInt64(&tcpOutput.dropMessageCount)
time.Sleep(time.Duration(100) * time.Millisecond)
}
c.Expect(dropcount, gs.Equals, int64(1))
close(inChan)
})
c.Specify("Overload queue shutdowns Heka", func() {
config.QueueFullAction = "shutdown"
config.QueueMaxBufferSize = uint64(1)
use_framing := false
config.UseFraming = &use_framing
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)
sigChan := globals.SigChan()
err := tcpOutput.Init(config)
c.Expect(err, gs.IsNil)
startOutput()
inChan <- pack
shutdownSignal := <-sigChan
c.Expect(shutdownSignal, gs.Equals, syscall.SIGINT)
close(inChan)
})
c.Specify("Overload queue blocks processing until packet is sent", func() {
config.QueueFullAction = "block"
config.QueueMaxBufferSize = uint64(1)
use_framing := false
config.UseFraming = &use_framing
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes()
oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)
err := tcpOutput.Init(config)
c.Expect(err, gs.IsNil)
startOutput()
inChan <- pack
msgcount := atomic.LoadInt64(&tcpOutput.dropMessageCount)
for x := 0; x < 5 && msgcount == 0; x++ {
msgcount = atomic.LoadInt64(&tcpOutput.dropMessageCount)
time.Sleep(time.Duration(100) * time.Millisecond)
}
c.Expect(atomic.LoadInt64(&tcpOutput.dropMessageCount), gs.Equals, int64(0))
c.Expect(atomic.LoadInt64(&tcpOutput.processMessageCount), gs.Equals, int64(0))
close(inChan)
})
})
}
func LogstreamerInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
here, _ := os.Getwd()
dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
c.Expect(tmpErr, gs.Equals, nil)
defer func() {
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
pConfig := NewPipelineConfig(globals)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// Specify localhost, but we're not really going to use the network.
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// Set up mock helper, runner, and pack supply channel.
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
c.Specify("A LogstreamerInput", func() {
lsInput := &LogstreamerInput{pConfig: pConfig}
lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
lsiConfig.LogDirectory = dirPath
lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
lsiConfig.Differentiator = []string{"logfile"}
lsiConfig.Priority = []string{"^Seq"}
c.Specify("w/ no translation map", func() {
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
c.Expect(len(lsInput.plugins), gs.Equals, 1)
// Create pool of packs.
numLines := 5 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs.
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
gomock.Any()).Times(numLines)
line := "boo hoo foo foo"
getRecCall.Return(len(line), []byte(line), nil)
getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
getRecCall.Return(0, make([]byte, 0), io.EOF)
deliverChan := make(chan []byte, 1)
deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
ith.MockDeliverer).Times(numLines)
deliverCall.Do(func(record []byte, del Deliverer) {
deliverChan <- record
})
ith.MockDeliverer.EXPECT().Done()
runOutChan := make(chan error, 1)
go func() {
err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
runOutChan <- err
}()
dur, _ := time.ParseDuration("5s")
timeout := time.After(dur)
timed := false
for x := 0; x < numLines; x++ {
select {
case record := <-deliverChan:
c.Expect(string(record), gs.Equals, line)
case <-timeout:
timed = true
x += numLines
}
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lsInput.Stop()
c.Expect(timed, gs.Equals, false)
c.Expect(<-runOutChan, gs.Equals, nil)
})
})
c.Specify("with a translation map", func() {
lsiConfig.Translation = make(ls.SubmatchTranslationMap)
lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)
c.Specify("allows len 1 translation map for 'missing'", func() {
lsiConfig.Translation["Seq"]["missing"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
})
c.Specify("doesn't allow len 1 map for other keys", func() {
lsiConfig.Translation["Seq"]["missin"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"A translation map with one entry ('Seq') must be specifying a "+
"'missing' key.")
})
})
})
}
func TestSocketInvalidOrigin(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
gomock.Any(), gomock.Any()).AnyTimes()
mckStat := NewMockStatistician(mockCtrl)
mckStat.EXPECT().Increment("client.socket.connect").AnyTimes()
mckStat.EXPECT().Timer("client.socket.lifespan", gomock.Any()).AnyTimes()
mckStat.EXPECT().Increment("client.socket.disconnect").AnyTimes()
mckStore := NewMockStore(mockCtrl)
mckRouter := NewMockRouter(mockCtrl)
testWithOrigin := func(allowedOrigins []string,
config *websocket.Config) error {
app := NewApplication()
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
app.SetStore(mckStore)
app.SetRouter(mckRouter)
sh := NewSocketHandler()
defer sh.Close()
sh.setApp(app)
sh.setOrigins(allowedOrigins)
pipe := newPipeListener()
if err := sh.listenWithConfig(listenerConfig{listener: pipe}); err != nil {
return err
}
sh.server = newServeWaiter(&http.Server{Handler: sh.ServeMux()})
app.SetSocketHandler(sh)
errChan := make(chan error, 1)
go sh.Start(errChan)
conn, err := dialSocketListener(pipe, config)
if err != nil {
return err
}
conn.Close()
return nil
}
location := &url.URL{Scheme: "https", Host: "example.com"}
badURL := &url.URL{Scheme: "!@#$", Host: "^&*-"}
tests := []struct {
name string
allowedOrigins []string
config *websocket.Config
err error
}{
{
"Should allow all origins if none are specified",
nil,
&websocket.Config{
Version: websocket.ProtocolVersionHybi13,
Location: location,
Origin: location,
},
nil,
},
{
"Should match multiple origins",
[]string{"https://example.com", "https://example.org", "https://example.net"},
&websocket.Config{
Version: websocket.ProtocolVersionHybi13,
Location: location,
Origin: &url.URL{Scheme: "https", Host: "example.net"},
},
nil,
},
{
"Should reject mismatched origins",
[]string{"https://example.com"},
&websocket.Config{
Version: websocket.ProtocolVersionHybi13,
Location: location,
Origin: &url.URL{Scheme: "http", Host: "example.org"},
},
websocket.ErrBadStatus, // 403 status code.
},
{
"Should reject malformed origins if some are specified",
[]string{"https://example.com"},
&websocket.Config{
Version: websocket.ProtocolVersionHybi13,
Location: location,
Origin: badURL, // Malformed URL.
},
websocket.ErrBadStatus, // 403 status code.
},
{
"Should ignore malformed origins if none are specified",
nil,
&websocket.Config{
Version: websocket.ProtocolVersionHybi13,
Location: location,
Origin: badURL,
},
nil,
},
{
"Should allow empty origins if none are specified",
nil,
&websocket.Config{
Version: websocket.ProtocolVersionHybi13,
Location: location,
Origin: &url.URL{},
},
nil,
},
{
"Should reject empty origins if some are specified",
[]string{"https://example.com"},
&websocket.Config{
Version: websocket.ProtocolVersionHybi13,
Location: location,
Origin: &url.URL{},
},
websocket.ErrBadStatus,
},
}
for _, test := range tests {
err := testWithOrigin(test.allowedOrigins, test.config)
if err != test.err {
t.Errorf("On test %s, got %#v; want %#v", test.name, err, test.err)
}
}
}
func SplitterRunnerSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
srConfig := CommonSplitterConfig{}
c.Specify("A SplitterRunner w/ HekaFramingSplitter", func() {
splitter := &HekaFramingSplitter{}
config := splitter.ConfigStruct().(*HekaFramingSplitterConfig)
useMsgBytes := true
srConfig.UseMsgBytes = &useMsgBytes
sr := NewSplitterRunner("HekaFramingSplitter", splitter, srConfig)
splitter.SetSplitterRunner(sr)
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
b, err := ioutil.ReadFile(filepath.Join(".", "testsupport", "multi.dat"))
c.Assume(err, gs.IsNil)
reader := makeMockReader(b)
c.Specify("correctly handles data at EOF", func() {
count, errCount, bytesRead, foundEOFCount,
remainingDataLength, finalRecordLength,
eofRecordLength := readRecordsFromStream(sr, reader, true)
c.Expect(errCount, gs.Equals, 0)
c.Expect(count, gs.Equals, 50)
c.Expect(foundEOFCount, gs.Equals, 50)
c.Expect(remainingDataLength, gs.Equals, 0)
c.Expect(finalRecordLength, gs.Equals, 215)
c.Expect(eofRecordLength, gs.Equals, 0)
c.Expect(bytesRead, gs.Equals, len(b))
})
c.Specify("correctly splits & unframes a protobuf stream", func() {
ir := NewMockInputRunner(ctrl)
sr.SetInputRunner(ir)
recycleChan := make(chan *PipelinePack, 1)
pack := NewPipelinePack(recycleChan)
recycleChan <- pack
numRecs := 50
ir.EXPECT().InChan().Times(numRecs).Return(recycleChan)
delCall := ir.EXPECT().Deliver(pack).Times(numRecs)
delCall.Do(func(pack *PipelinePack) {
pack.Recycle()
})
for err == nil {
err = sr.SplitStream(reader, nil)
}
c.Expect(err, gs.Equals, io.EOF)
})
c.Specify("correctly handles appends after EOF", func() {
half := len(b) / 2
reader := makeMockReader(b[:half])
totalBytesRead := 0
count, errCount, bytesRead, foundEOFCount, _, finalRecordLength,
eofRecordLength := readRecordsFromStream(sr, reader, false)
totalBytesRead += bytesRead
c.Expect(errCount, gs.Equals, 0)
c.Expect(count, gs.Equals, 25)
c.Expect(foundEOFCount, gs.Equals, 25)
c.Expect(finalRecordLength, gs.Equals, 215)
c.Expect(eofRecordLength, gs.Equals, 0)
c.Expect(bytesRead <= half, gs.IsTrue)
reader.Append(b[half:])
count, errCount, bytesRead, foundEOFCount,
remainingDataLength, finalRecordLength,
eofRecordLength := readRecordsFromStream(sr, reader, true)
totalBytesRead += bytesRead
c.Expect(errCount, gs.Equals, 0)
c.Expect(count, gs.Equals, 25)
c.Expect(foundEOFCount, gs.Equals, 25)
c.Expect(remainingDataLength, gs.Equals, 0)
c.Expect(finalRecordLength, gs.Equals, 215)
c.Expect(eofRecordLength, gs.Equals, 0)
c.Expect(totalBytesRead, gs.Equals, len(b))
})
c.Specify("reuse on another stream without GetRemainingData", func() {
// Test the case where we reuse the same SplitterRunner on
// two different readers, and we do not call GetRemainingData before
// using the second reader.
half := len(b) / 2
reader1 := makeMockReader(b[:half])
count, errCount, bytesRead, foundEOFCount, _, finalRecordLength,
eofRecordLength := readRecordsFromStream(sr, reader1, false)
c.Expect(errCount, gs.Equals, 0)
c.Expect(count, gs.Equals, 25)
c.Expect(foundEOFCount, gs.Equals, 25)
c.Expect(finalRecordLength, gs.Equals, 215)
c.Expect(eofRecordLength, gs.Equals, 0)
leftovers := half - bytesRead
c.Expect(leftovers > 0, gs.IsTrue)
reader2 := makeMockReader(b)
// Don't call GetRemainingData before using sr on a new stream
count, errCount, bytesRead, foundEOFCount, remainingDataLength, finalRecordLength,
eofRecordLength := readRecordsFromStream(sr, reader2, true)
c.Expect(errCount, gs.Equals, 0)
c.Expect(count, gs.Equals, 50)
c.Expect(foundEOFCount, gs.Equals, 50)
c.Expect(remainingDataLength, gs.Equals, 0)
c.Expect(finalRecordLength, gs.Equals, 215)
c.Expect(eofRecordLength, gs.Equals, 0)
// sr misreports the "remaining data" piece from reader1 as being
// read from reader2
c.Expect(bytesRead, gs.Equals, len(b)+leftovers)
})
c.Specify("reuse on another stream with reset", func() {
// Test the case where we reuse the same SplitterRunner on
// two different readers, but we call GetRemainingData before using
// the second reader.
half := len(b) / 2
reader1 := makeMockReader(b[:half])
count, errCount, bytesRead, foundEOFCount, _, finalRecordLength,
eofRecordLength := readRecordsFromStream(sr, reader1, false)
c.Expect(errCount, gs.Equals, 0)
c.Expect(count, gs.Equals, 25)
c.Expect(foundEOFCount, gs.Equals, 25)
c.Expect(finalRecordLength, gs.Equals, 215)
c.Expect(eofRecordLength, gs.Equals, 0)
leftovers := half - bytesRead
c.Expect(leftovers > 0, gs.IsTrue)
reader2 := makeMockReader(b)
// Call GetRemainingData before using sr on a new stream
sr.GetRemainingData()
count, errCount, bytesRead, foundEOFCount, remainingDataLength, finalRecordLength,
eofRecordLength := readRecordsFromStream(sr, reader2, true)
c.Expect(errCount, gs.Equals, 0)
c.Expect(count, gs.Equals, 50)
c.Expect(foundEOFCount, gs.Equals, 50)
c.Expect(remainingDataLength, gs.Equals, 0)
c.Expect(finalRecordLength, gs.Equals, 215)
c.Expect(eofRecordLength, gs.Equals, 0)
// Now we see the correct number of bytes being read.
c.Expect(bytesRead, gs.Equals, len(b))
})
})
c.Specify("A SplitterRunner w/ TokenSplitter", func() {
splitter := &TokenSplitter{}
config := splitter.ConfigStruct().(*TokenSplitterConfig)
c.Specify("sets readPos to 0 when read returns ErrShortBuffer", func() {
config.Delimiter = "\t"
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
sr := NewSplitterRunner("TokenSplitter", splitter, srConfig)
b := make([]byte, message.MAX_RECORD_SIZE+1)
reader := bytes.NewReader(b)
var n int
var record []byte
for err == nil {
n, record, err = sr.GetRecordFromStream(reader)
}
c.Expect(n, gs.Equals, int(message.MAX_RECORD_SIZE))
c.Expect(len(record), gs.Equals, 0)
c.Expect(err, gs.Equals, io.ErrShortBuffer)
c.Expect(sr.readPos, gs.Equals, 0)
c.Expect(sr.scanPos, gs.Equals, 0)
})
c.Specify("checks if splitter honors 'deliver_incomplete_final' setting", func() {
config.Count = 4
numRecs := 10
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
packSupply := make(chan *PipelinePack, 1)
pack := NewPipelinePack(packSupply)
packSupply <- pack
ir := NewMockInputRunner(ctrl)
// ir.EXPECT().InChan().Return(packSupply).Times(numRecs)
// ir.EXPECT().Name().Return("foo").Times(numRecs)
ir.EXPECT().InChan().Return(packSupply).AnyTimes()
ir.EXPECT().Name().Return("foo").AnyTimes()
incompleteFinal := true
srConfig.IncompleteFinal = &incompleteFinal
sr := NewSplitterRunner("TokenSplitter", splitter, srConfig)
sr.ir = ir
rExpected := []byte("test1\ntest12\ntest123\npartial\n")
buf := bytes.Repeat(rExpected, numRecs)
buf = buf[:len(buf)-1] // 40 lines separated by 39 newlines
reader := bytes.NewReader(buf)
mockDel := NewMockDeliverer(ctrl)
delCall := mockDel.EXPECT().Deliver(gomock.Any()).AnyTimes()
i := 0
delCall.Do(func(pack *PipelinePack) {
i++
if i < numRecs {
c.Expect(pack.Message.GetPayload(), gs.Equals, string(rExpected))
} else {
c.Expect(pack.Message.GetPayload(), gs.Equals,
string(rExpected[:len(rExpected)-1]))
}
pack.Recycle()
})
c.Specify("via SplitStream", func() {
for err == nil {
err = sr.SplitStream(reader, mockDel)
}
c.Expect(err, gs.Equals, io.EOF)
c.Expect(i, gs.Equals, numRecs)
})
c.Specify("via SplitBytes", func() {
seekPos, err := sr.SplitBytes(buf, mockDel)
c.Assume(err, gs.IsNil)
c.Expect(seekPos, gs.Equals, len(buf))
c.Expect(i, gs.Equals, numRecs)
})
})
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups.
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel.
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return.
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
bytesChan := make(chan []byte, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init.
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Set up relevant mocks.
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.MockSplitterRunner = NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a text message", func() {
// Create a channel to send data to the input. Drop a message on
// there and close the channel.
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "This is a message")
close(streamChan)
err = <-errChan
})
c.Specify("consumes a protobuf encoded message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, string(msgBody))
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
// Expect the close and the InChan calls.
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func TestSocketOrigin(t *testing.T) {
var err error
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
gomock.Any(), gomock.Any()).AnyTimes()
mckStat := NewMockStatistician(mockCtrl)
mckStore := NewMockStore(mockCtrl)
mckRouter := NewMockRouter(mockCtrl)
app := NewApplication()
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
app.SetStore(mckStore)
app.SetRouter(mckRouter)
sh := NewSocketHandler()
defer sh.Close()
sh.setApp(app)
pipe := newPipeListener()
defer pipe.Close()
if err := sh.listenWithConfig(listenerConfig{listener: pipe}); err != nil {
t.Fatalf("Error setting listener: %s", err)
}
sh.server = newServeWaiter(&http.Server{Handler: sh.ServeMux()})
app.SetSocketHandler(sh)
errChan := make(chan error, 1)
go sh.Start(errChan)
uaid := "5e1e5984569c4f00bf4bea47754a6403"
gomock.InOrder(
mckStat.EXPECT().Increment("client.socket.connect"),
mckStore.EXPECT().CanStore(0).Return(true),
mckRouter.EXPECT().Register(uaid),
mckStat.EXPECT().Increment("updates.client.hello"),
mckStore.EXPECT().FetchAll(uaid, gomock.Any()).Return(nil, nil, nil),
mckStat.EXPECT().Timer("client.flush", gomock.Any()),
mckRouter.EXPECT().Unregister(uaid),
mckStat.EXPECT().Timer("client.socket.lifespan", gomock.Any()),
mckStat.EXPECT().Increment("client.socket.disconnect"),
)
origin := &url.URL{Scheme: "https", Host: "example.com"}
conn, err := dialSocketListener(pipe, &websocket.Config{
Location: origin,
Origin: origin,
Version: websocket.ProtocolVersionHybi13,
})
if err != nil {
t.Fatalf("Error dialing origin: %s", err)
}
defer conn.Close()
err = websocket.JSON.Send(conn, struct {
Type string `json:"messageType"`
DeviceID string `json:"uaid"`
ChannelIDs []string `json:"channelIDs"`
}{"hello", uaid, []string{}})
if err != nil {
t.Fatalf("Error writing client handshake: %s", err)
}
reply := new(HelloReply)
if err = websocket.JSON.Receive(conn, reply); err != nil {
t.Fatalf("Error reading server handshake: %s", err)
}
if reply.DeviceID != uaid {
t.Fatalf("Mismatched device ID: got %q; want %q", reply.DeviceID, uaid)
}
worker, workerConnected := app.GetWorker(uaid)
if !workerConnected {
t.Fatalf("Missing worker for device ID %q", uaid)
}
workerOrigin := worker.Origin()
if expectedOrigin := origin.String(); workerOrigin != expectedOrigin {
t.Errorf("Mismatched origins: got %q; want %q",
workerOrigin, expectedOrigin)
}
}
func HttpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`
c.Specify("A HttpInput", func() {
httpInput := HttpInput{}
ith := new(plugins_ts.InputTestHelper)
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
runOutputChan := make(chan string, 1)
startInput := func() {
go func() {
err := httpInput.Run(ith.MockInputRunner, ith.MockHelper)
var runOutput string
if err != nil {
runOutput = err.Error()
}
runOutputChan <- runOutput
}()
}
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
config := httpInput.ConfigStruct().(*HttpInputConfig)
c.Specify("short circuits packs into the router", func() {
config.Url = "http://localhost:9876/"
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
err := httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// We need for the pipeline to finish up
time.Sleep(50 * time.Millisecond)
})
c.Specify("with a decoder", func() {
decoderName := "TestDecoder"
config.DecoderName = decoderName
c.Specify("honors time ticker to flush", func() {
// Spin up a http server
server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876)
c.Expect(err, gs.IsNil)
go server.Start("/")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9876/"
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Stub out the DecoderRunner input channel so that we can
// inspect bytes later on
dRunnerInChan := make(chan *PipelinePack, 1)
mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)
ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// We need for the pipeline to finish up
time.Sleep(50 * time.Millisecond)
})
c.Specify("supports configuring HTTP Basic Authentication", func() {
// Spin up a http server which expects username "user" and password "password"
server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875)
c.Expect(err, gs.IsNil)
go server.Start("/BasicAuthTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9875/BasicAuthTest"
config.User = "******"
config.Password = "******"
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Stub out the DecoderRunner input channel so that we can
// inspect bytes later on
dRunnerInChan := make(chan *PipelinePack, 1)
mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)
ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// we expect a statuscode 200 (i.e. success)
pack := <-dRunnerInChan
statusCode, ok := pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
// We need for the pipeline to finish up
time.Sleep(50 * time.Millisecond)
})
c.Specify("supports configuring a different HTTP method", func() {
// Spin up a http server which expects requests with method "POST"
server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874)
c.Expect(err, gs.IsNil)
go server.Start("/PostTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9874/PostTest"
config.Method = "POST"
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Stub out the DecoderRunner input channel so that we can
// inspect bytes later on
dRunnerInChan := make(chan *PipelinePack, 1)
mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)
ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// we expect a statuscode 200 (i.e. success)
pack := <-dRunnerInChan
statusCode, ok := pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
// We need for the pipeline to finish up
time.Sleep(50 * time.Millisecond)
})
c.Specify("supports configuring HTTP headers", func() {
// Spin up a http server which expects requests with method "POST"
server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873)
c.Expect(err, gs.IsNil)
go server.Start("/HeadersTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9873/HeadersTest"
config.Headers = map[string]string{"Accept": "text/plain"}
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Stub out the DecoderRunner input channel so that we can
// inspect bytes later on
dRunnerInChan := make(chan *PipelinePack, 1)
mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)
ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// we expect a statuscode 200 (i.e. success)
pack := <-dRunnerInChan
statusCode, ok := pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
// We need for the pipeline to finish up
time.Sleep(50 * time.Millisecond)
})
c.Specify("supports configuring a request body", func() {
// Spin up a http server that echoes back the request body
server, err := plugins_ts.NewHttpBodyServer("localhost", 9872)
c.Expect(err, gs.IsNil)
go server.Start("/BodyTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9872/BodyTest"
config.Method = "POST"
config.Body = json_post
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Stub out the DecoderRunner input channel so that we can
// inspect bytes later on
dRunnerInChan := make(chan *PipelinePack, 1)
mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)
ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
pack := <-dRunnerInChan
c.Expect(*pack.Message.Payload, gs.Equals, json_post)
// We need for the pipeline to finish up
time.Sleep(50 * time.Millisecond)
})
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any())
httpInput.Stop()
runOutput := <-runOutputChan
c.Expect(runOutput, gs.Equals, "")
})
})
}
func ProcessInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
c.Specify("A ProcessInput", func() {
pInput := ProcessInput{}
config := pInput.ConfigStruct().(*ProcessInputConfig)
config.Command = make(map[string]cmdConfig)
ith.MockHelper.EXPECT().Hostname().Return(pConfig.Hostname())
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
errChan := make(chan error)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
decChan := make(chan func(*PipelinePack), 1)
setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
setDecCall.Do(func(dec func(*PipelinePack)) {
decChan <- dec
})
bytesChan := make(chan []byte, 1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
ith.MockDeliverer).Return(nil)
splitCall.Do(func(r io.Reader, del Deliverer) {
bytes, err := ioutil.ReadAll(r)
c.Assume(err, gs.IsNil)
bytesChan <- bytes
})
ith.MockDeliverer.EXPECT().Done()
c.Specify("using stdout", func() {
ith.MockInputRunner.EXPECT().NewDeliverer("stdout").Return(ith.MockDeliverer)
ith.MockInputRunner.EXPECT().NewSplitterRunner("stdout").Return(
ith.MockSplitterRunner)
c.Specify("reads a message from ProcessInput", func() {
pInput.SetName("SimpleTest")
// Note that no working directory is explicitly specified.
config.Command["0"] = cmdConfig{
Bin: PROCESSINPUT_TEST1_CMD,
Args: PROCESSINPUT_TEST1_CMD_ARGS,
}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
actual := <-bytesChan
c.Expect(string(actual), gs.Equals, PROCESSINPUT_TEST1_OUTPUT+"\n")
dec := <-decChan
dec(ith.Pack)
fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")
pInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("can pipe multiple commands together", func() {
pInput.SetName("PipedCmd")
// Note that no working directory is explicitly specified.
config.Command["0"] = cmdConfig{
Bin: PROCESSINPUT_PIPE_CMD1,
Args: PROCESSINPUT_PIPE_CMD1_ARGS,
}
config.Command["1"] = cmdConfig{
Bin: PROCESSINPUT_PIPE_CMD2,
Args: PROCESSINPUT_PIPE_CMD2_ARGS,
}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
actual := <-bytesChan
c.Expect(string(actual), gs.Equals, PROCESSINPUT_PIPE_OUTPUT+"\n")
dec := <-decChan
dec(ith.Pack)
fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")
pInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
c.Specify("using stderr", func() {
ith.MockInputRunner.EXPECT().NewDeliverer("stderr").Return(ith.MockDeliverer)
ith.MockInputRunner.EXPECT().NewSplitterRunner("stderr").Return(
ith.MockSplitterRunner)
c.Specify("handles bad arguments", func() {
pInput.SetName("BadArgs")
config.ParseStdout = false
config.ParseStderr = true
// Note that no working directory is explicitly specified.
config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
expectedErr := fmt.Errorf(
"BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]")
ith.MockInputRunner.EXPECT().LogError(expectedErr)
go func() {
errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
// Error message differs by platform, but we at least wait
// until we get it.
<-bytesChan
pInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
}
func HttpListenInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
httpListenInput := HttpListenInput{}
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
splitter := &TokenSplitter{} // Not actually used.
errChan := make(chan error, 1)
startInput := func() {
go func() {
err := httpListenInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
}
config := httpListenInput.ConfigStruct().(*HttpListenInputConfig)
config.Address = "127.0.0.1:58325"
c.Specify("A HttpListenInput", func() {
startedChan := make(chan bool, 1)
defer close(startedChan)
ts := httptest.NewUnstartedServer(nil)
httpListenInput.starterFunc = func(hli *HttpListenInput) error {
ts.Start()
startedChan <- true
return nil
}
// These EXPECTs imply that every spec below will send exactly one
// HTTP request to the input.
ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)
decChan := make(chan func(*PipelinePack), 1)
feedDecorator := func(decorator func(*PipelinePack)) {
decChan <- decorator
}
setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
setDecCall.Do(feedDecorator)
streamChan := make(chan io.Reader, 1)
feedStream := func(r io.Reader) {
streamChan <- r
}
getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
gomock.Any()).Do(feedStream)
bytesChan := make(chan []byte, 1)
deliver := func(msgBytes []byte, del Deliverer) {
bytesChan <- msgBytes
}
ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()
c.Specify("Adds query parameters to the message pack as fields", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
getRecCall.Return(0, make([]byte, 0), io.EOF)
startInput()
<-startedChan
resp, err := http.Get(ts.URL + "/?test=Hello%20World")
resp.Body.Close()
c.Assume(err, gs.IsNil)
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("test")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "Hello World")
})
c.Specify("Add custom headers", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
getRecCall.Return(0, make([]byte, 0), io.EOF)
startInput()
<-startedChan
resp, err := http.Get(ts.URL)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
// Verify headers are there
eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
c.Expect(eq, gs.IsTrue)
eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
c.Expect(eq, gs.IsTrue)
})
c.Specify("Request body is sent as record", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
body := "1+2"
getRecCall.Return(0, []byte(body), io.EOF)
startInput()
<-startedChan
deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
nil)
deliverCall.Do(deliver)
resp, err := http.Post(ts.URL, "text/plain", strings.NewReader(body))
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "1+2")
})
c.Specify("Add request headers as fields", func() {
config.RequestHeaders = []string{
"X-REQUEST-ID",
}
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
getRecCall.Return(0, make([]byte, 0), io.EOF)
startInput()
<-startedChan
client := &http.Client{}
req, err := http.NewRequest("GET", ts.URL, nil)
req.Header.Add("X-REQUEST-ID", "12345")
resp, err := client.Do(req)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("X-REQUEST-ID")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "12345")
})
c.Specify("Add other request properties as fields", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
getRecCall.Return(0, make([]byte, 0), io.EOF)
startInput()
<-startedChan
client := &http.Client{}
req, err := http.NewRequest("GET", ts.URL+"/foo/bar?baz=2", nil)
c.Assume(err, gs.IsNil)
req.Host = "incoming.example.com:8080"
resp, err := client.Do(req)
resp.Body.Close()
c.Assume(err, gs.IsNil)
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("Host")
c.Assume(ok, gs.IsTrue)
// Host should not include port number.
c.Expect(fieldValue, gs.Equals, "incoming.example.com")
fieldValue, ok = ith.Pack.Message.GetFieldValue("Path")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "/foo/bar")
hostname, err := os.Hostname()
c.Assume(err, gs.IsNil)
c.Expect(*ith.Pack.Message.Hostname, gs.Equals, hostname)
c.Expect(*ith.Pack.Message.EnvVersion, gs.Equals, "1")
fieldValue, ok = ith.Pack.Message.GetFieldValue("RemoteAddr")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue != nil, gs.IsTrue)
fieldValueStr, ok := fieldValue.(string)
c.Assume(ok, gs.IsTrue)
c.Expect(len(fieldValueStr) > 0, gs.IsTrue)
// Per the `Request` docs, this should be an IP address:
// http://golang.org/pkg/net/http/#Request
ip := net.ParseIP(fieldValueStr)
c.Expect(ip != nil, gs.IsTrue)
})
ts.Close()
httpListenInput.Stop()
err := <-errChan
c.Expect(err, gs.IsNil)
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockDRunner := NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
c.Specify("with a valid setup and no decoder", func() {
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a message", func() {
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
})
c.Specify("with a valid setup using a decoder", func() {
decoderName := "defaultDecoder"
config.Decoder = decoderName
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
// Mock up our default decoder runner and decoder.
ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
decCall.Return(mockDRunner, true)
mockDecoder := NewMockDecoder(ctrl)
mockDRunner.EXPECT().Decoder().Return(mockDecoder)
c.Specify("consumes a message", func() {
packs := []*PipelinePack{ith.Pack}
mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
c.Specify("consumes a serialized message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the decoded pack
mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
// Ignore leading 5 bytes of encoded message as thats the header
c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close
ug.Add(1)
// Expect the close and the InChan calls
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func TestEndpointInvalidParams(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
gomock.Any(), gomock.Any()).AnyTimes()
mckStat := NewMockStatistician(mockCtrl)
Convey("Invalid update parameters", t, func() {
app := NewApplication()
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
eh := NewEndpointHandler()
eh.setApp(app)
eh.setMaxDataLen(512)
app.SetEndpointHandler(eh)
Convey("Should require PUT requests", func() {
resp := httptest.NewRecorder()
req := &http.Request{
Method: "POST",
Header: http.Header{},
URL: &url.URL{Path: "/update/123"},
}
mckStat.EXPECT().Increment("updates.appserver.invalid")
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 405)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, `"Method Not Allowed"`)
})
Convey("Should reject negative versions", func() {
vals := make(url.Values)
vals.Set("version", "-1")
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{},
URL: &url.URL{Path: "/update/123"},
Body: formReader(vals),
}
mckStat.EXPECT().Increment("updates.appserver.invalid")
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 400)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, `"Invalid Version"`)
})
Convey("Should reject invalid versions", func() {
vals := make(url.Values)
vals.Set("version", "abc")
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{},
URL: &url.URL{Path: "/update/123"},
Body: formReader(vals),
}
mckStat.EXPECT().Increment("updates.appserver.invalid")
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 400)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, `"Invalid Version"`)
})
Convey("Should reject oversized payloads", func() {
vals := make(url.Values)
vals.Set("data", randomText(eh.maxDataLen+1))
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{},
URL: &url.URL{Path: "/update/123"},
Body: formReader(vals),
}
mckStat.EXPECT().Increment("updates.appserver.toolong")
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 413)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, `"Data exceeds max length of 512 bytes"`)
})
})
}
func Test_GCMSend(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(),
gomock.Any()).AnyTimes()
mckStat := &TestMetrics{}
mckStat.Init(nil, nil)
mckStore := NewMockStore(mockCtrl)
mckEndHandler := NewMockHandler(mockCtrl)
mckGCMClient := &mockGCMClient{t: t}
Convey("GCM Proprietary Ping", t, func() {
uaid := "deadbeef00000000000000000000"
vers := int64(1)
data := "i'm a little teapot"
fakeConnect := []byte(`{"regid":"testing"}`)
app := NewApplication()
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
app.SetStore(mckStore)
app.SetEndpointHandler(mckEndHandler)
mckGCMClient.reset()
headers := make(http.Header, 1)
headers.Set("Retry-After", "1")
mckGCMClient.reply = &http.Response{
Body: respBody("Ok"),
StatusCode: 503,
Header: headers,
}
mckStore.EXPECT().FetchPing(uaid).Return(fakeConnect, nil)
testGcm := &GCMPing{
logger: app.Logger(),
metrics: mckStat,
store: mckStore,
apiKey: "test_api_key",
}
conf := testGcm.ConfigStruct()
err := testGcm.Init(app, conf)
So(err, ShouldBeNil)
testGcm.ReplaceClient(mckGCMClient)
// For this test, we first respond with a fake 503 message that
// requests a "retry after", we follow up with a second message
// that returns success.
//
// Not sure why but when using gomock, calls to Increment are
// not being registered, so falling back to older testMetrics
// object.
ok, err := testGcm.Send(uaid, vers, data)
So(err, ShouldBeNil)
So(ok, ShouldEqual, true)
So(mckStat.Counters["ping.gcm.retry"], ShouldEqual, 1)
So(mckStat.Counters["ping.gcm.success"], ShouldEqual, 1)
})
}
func TestEndpointDelivery(t *testing.T) {
useMockFuncs()
defer useStdFuncs()
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
gomock.Any(), gomock.Any()).AnyTimes()
mckStat := NewMockStatistician(mockCtrl)
mckStore := NewMockStore(mockCtrl)
mckRouter := NewMockRouter(mockCtrl)
mckWorker := NewMockWorker(mockCtrl)
Convey("Update delivery", t, func() {
app := NewApplication()
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
app.SetStore(mckStore)
app.SetRouter(mckRouter)
Convey("Should attempt local delivery if `AlwaysRoute` is disabled", func() {
eh := NewEndpointHandler()
eh.setApp(app)
app.SetEndpointHandler(eh)
Convey("Should route updates if the device is not connected", func() {
uaid := "f7e9fc483f7344c398701b6fa0e85e4f"
chid := "737b7a0d25674be4bb184f015fce02cf"
gomock.InOrder(
mckStat.EXPECT().Increment("updates.routed.outgoing"),
mckRouter.EXPECT().Route(nil, uaid, chid, int64(3), timeNow().UTC(),
"", "").Return(true, nil),
mckStat.EXPECT().Increment("router.broadcast.hit"),
mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
mckStat.EXPECT().Increment("updates.appserver.received"),
)
ok := eh.deliver(nil, uaid, chid, 3, "", "")
So(ok, ShouldBeTrue)
})
// A routing failure still stores the alert for potential
// client delivery. We should only return 404 for absolute
// failures (where the endpoint is no longer valid
Convey("Should return a 202 if routing fails", func() {
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{HeaderID: {"reqID"}},
URL: &url.URL{Path: "/update/123"},
Body: formReader(url.Values{"version": {"1"}}),
}
gomock.InOrder(
mckStore.EXPECT().KeyToIDs("123").Return("123", "456", nil),
mckStat.EXPECT().Increment("updates.appserver.incoming"),
mckStore.EXPECT().Update("123", "456", int64(1)).Return(nil),
mckStat.EXPECT().Increment("updates.routed.outgoing"),
mckRouter.EXPECT().Route(nil, "123", "456", int64(1),
gomock.Any(), "reqID", "").Return(false, nil),
mckStat.EXPECT().Increment("router.broadcast.miss"),
mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
mckStat.EXPECT().Increment("updates.appserver.rejected"),
)
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 202)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, "{}")
})
Convey("Should return a 404 if local delivery fails", func() {
uaid := "9e98d6415d8e4fd099ab1bad7178f750"
chid := "0eecf572e99f4d508666d8da6c0b15a9"
app.AddWorker(uaid, mckWorker)
gomock.InOrder(
mckWorker.EXPECT().Send(chid, int64(3), "").Return(
errors.New("client gone")),
mckStat.EXPECT().Increment("updates.appserver.rejected"),
)
ok := eh.deliver(nil, uaid, chid, int64(3), "", "")
So(ok, ShouldBeFalse)
})
Convey("Should return an error if storage is unavailable", func() {
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{},
URL: &url.URL{Path: "/update/123"},
Body: formReader(url.Values{"version": {"2"}}),
}
updateErr := ErrInvalidChannel
gomock.InOrder(
mckStore.EXPECT().KeyToIDs("123").Return("123", "456", nil),
mckStat.EXPECT().Increment("updates.appserver.incoming"),
mckStore.EXPECT().Update("123", "456", int64(2)).Return(updateErr),
mckStat.EXPECT().Increment("updates.appserver.error"),
)
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, updateErr.Status())
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, `"Could not update channel version"`)
})
})
Convey("Should always route updates if `AlwaysRoute` is enabled", func() {
eh := NewEndpointHandler()
eh.setApp(app)
eh.alwaysRoute = true
app.SetEndpointHandler(eh)
uaid := "6952a68ee0e7444ebc54f935c4444b13"
app.AddWorker(uaid, mckWorker)
chid := "b7ede546585f4cc9b95e9340e3406951"
version := int64(1)
data := "Happy, happy, joy, joy!"
Convey("And router delivery fails, local succeeds", func() {
gomock.InOrder(
mckStat.EXPECT().Increment("updates.routed.outgoing"),
mckRouter.EXPECT().Route(nil, uaid, chid, version,
gomock.Any(), "", data).Return(false, nil),
mckStat.EXPECT().Increment("router.broadcast.miss"),
mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
mckWorker.EXPECT().Send(chid, version, data).Return(nil),
mckStat.EXPECT().Increment("updates.appserver.received"),
)
ok := eh.deliver(nil, uaid, chid, version, "", data)
So(ok, ShouldBeTrue)
})
Convey("And router delivery succeeds, local succeeds", func() {
gomock.InOrder(
mckStat.EXPECT().Increment("updates.routed.outgoing"),
mckRouter.EXPECT().Route(nil, uaid, chid, version,
gomock.Any(), "", data).Return(true, nil),
mckStat.EXPECT().Increment("router.broadcast.hit"),
mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
mckWorker.EXPECT().Send(chid, version, data).Return(nil),
mckStat.EXPECT().Increment("updates.appserver.received"),
)
ok := eh.deliver(nil, uaid, chid, version, "", data)
So(ok, ShouldBeTrue)
})
Convey("And router delivery succeeds, local fails", func() {
gomock.InOrder(
mckStat.EXPECT().Increment("updates.routed.outgoing"),
mckRouter.EXPECT().Route(nil, uaid, chid, version,
gomock.Any(), "", data).Return(true, nil),
mckStat.EXPECT().Increment("router.broadcast.hit"),
mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
mckWorker.EXPECT().Send(chid, version, data).Return(
errors.New("client gone")),
mckStat.EXPECT().Increment("updates.appserver.received"),
)
ok := eh.deliver(nil, uaid, chid, version, "", data)
So(ok, ShouldBeTrue)
})
Convey("And router/local delivery fails", func() {
gomock.InOrder(
mckStat.EXPECT().Increment("updates.routed.outgoing"),
mckRouter.EXPECT().Route(nil, uaid, chid, version,
gomock.Any(), "", data).Return(false, nil),
mckStat.EXPECT().Increment("router.broadcast.miss"),
mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
mckWorker.EXPECT().Send(chid, version, data).Return(
errors.New("client gone")),
mckStat.EXPECT().Increment("updates.appserver.rejected"),
)
ok := eh.deliver(nil, uaid, chid, version, "", data)
So(ok, ShouldBeFalse)
})
})
})
}
func HttpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`
c.Specify("A HttpInput", func() {
httpInput := HttpInput{}
ith := new(plugins_ts.InputTestHelper)
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
runOutputChan := make(chan error, 1)
startInput := func() {
go func() {
runOutputChan <- httpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
}
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// These assume that every sub-spec starts the input.
config := httpInput.ConfigStruct().(*HttpInputConfig)
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
ith.MockHelper.EXPECT().Hostname().Return("hekatests.example.com")
// These assume that every sub-spec makes exactly one HTTP request.
ith.MockInputRunner.EXPECT().NewSplitterRunner("0").Return(ith.MockSplitterRunner)
getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
getRecCall.Return(len(json_post), []byte(json_post), io.EOF)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
decChan := make(chan func(*PipelinePack), 1)
packDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
packDecCall.Do(func(dec func(*PipelinePack)) {
decChan <- dec
})
ith.MockSplitterRunner.EXPECT().DeliverRecord([]byte(json_post), nil)
ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()
splitter := &TokenSplitter{} // not actually used
ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)
c.Specify("honors time ticker to flush", func() {
// Spin up a http server.
server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876)
c.Expect(err, gs.IsNil)
go server.Start("/")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9876/"
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// Getting the decorator means we've made our HTTP request.
<-decChan
})
c.Specify("supports configuring HTTP Basic Authentication", func() {
// Spin up a http server which expects username "user" and password "password"
server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875)
c.Expect(err, gs.IsNil)
go server.Start("/BasicAuthTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9875/BasicAuthTest"
config.User = "******"
config.Password = "******"
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
dec := <-decChan
dec(ith.Pack)
// we expect a statuscode 200 (i.e. success)
statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
})
c.Specify("supports configuring a different HTTP method", func() {
// Spin up a http server which expects requests with method "POST"
server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874)
c.Expect(err, gs.IsNil)
go server.Start("/PostTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9874/PostTest"
config.Method = "POST"
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
dec := <-decChan
dec(ith.Pack)
// we expect a statuscode 200 (i.e. success)
statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
})
c.Specify("supports configuring HTTP headers", func() {
// Spin up a http server which expects requests with method "POST"
server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873)
c.Expect(err, gs.IsNil)
go server.Start("/HeadersTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9873/HeadersTest"
config.Headers = map[string]string{"Accept": "text/plain"}
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
dec := <-decChan
dec(ith.Pack)
// we expect a statuscode 200 (i.e. success)
statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
})
c.Specify("supports configuring a request body", func() {
// Spin up a http server that echoes back the request body
server, err := plugins_ts.NewHttpBodyServer("localhost", 9872)
c.Expect(err, gs.IsNil)
go server.Start("/BodyTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9872/BodyTest"
config.Method = "POST"
config.Body = json_post
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
respBodyChan := make(chan []byte, 1)
getRecCall.Do(func(r io.Reader) {
respBody := make([]byte, len(config.Body))
n, err := r.Read(respBody)
c.Expect(n, gs.Equals, len(config.Body))
c.Expect(err, gs.Equals, io.EOF)
respBodyChan <- respBody
})
startInput()
tickChan <- time.Now()
respBody := <-respBodyChan
c.Expect(string(respBody), gs.Equals, json_post)
})
httpInput.Stop()
runOutput := <-runOutputChan
c.Expect(runOutput, gs.IsNil)
})
}
func InputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
var wg sync.WaitGroup
errChan := make(chan error, 1)
tickChan := make(chan time.Time)
defer func() {
close(tickChan)
close(errChan)
ctrl.Finish()
}()
pConfig := NewPipelineConfig(nil)
c.Specify("A SandboxInput", func() {
input := new(SandboxInput)
input.SetPipelineConfig(pConfig)
ith := new(plugins_ts.InputTestHelper)
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.Pack = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- ith.Pack
startInput := func() {
wg.Add(1)
go func() {
errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
wg.Done()
}()
}
c.Specify("test a polling input", func() {
tickChan := time.Tick(10 * time.Millisecond)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(2)
ith.MockInputRunner.EXPECT().LogError(fmt.Errorf("failure message"))
var cnt int
ith.MockInputRunner.EXPECT().Inject(gomock.Any()).Do(
func(pack *PipelinePack) {
switch cnt {
case 0:
c.Expect(pack.Message.GetPayload(), gs.Equals, "line 1")
case 1:
c.Expect(pack.Message.GetPayload(), gs.Equals, "line 3")
input.Stop()
}
cnt++
ith.PackSupply <- pack
}).Times(2)
config := input.ConfigStruct().(*sandbox.SandboxConfig)
config.ScriptFilename = "../lua/testsupport/input.lua"
err := input.Init(config)
c.Assume(err, gs.IsNil)
startInput()
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
c.Expect(input.processMessageCount, gs.Equals, int64(2))
c.Expect(input.processMessageFailures, gs.Equals, int64(1))
c.Expect(input.processMessageBytes, gs.Equals, int64(72))
})
c.Specify("run once input", func() {
var tickChan <-chan time.Time
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(1)
ith.MockInputRunner.EXPECT().LogMessage("single run completed")
ith.MockInputRunner.EXPECT().Inject(gomock.Any()).Do(
func(pack *PipelinePack) {
c.Expect(pack.Message.GetPayload(), gs.Equals, "line 1")
ith.PackSupply <- pack
}).Times(1)
config := input.ConfigStruct().(*sandbox.SandboxConfig)
config.ScriptFilename = "../lua/testsupport/input.lua"
err := input.Init(config)
c.Assume(err, gs.IsNil)
startInput()
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
c.Expect(input.processMessageCount, gs.Equals, int64(1))
c.Expect(input.processMessageBytes, gs.Equals, int64(36))
})
c.Specify("exit with error", func() {
tickChan := make(chan time.Time)
defer close(tickChan)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
ith.MockInputRunner.EXPECT().LogError(fmt.Errorf("process_message() ../lua/testsupport/input_error.lua:2: boom"))
config := input.ConfigStruct().(*sandbox.SandboxConfig)
config.ScriptFilename = "../lua/testsupport/input_error.lua"
err := input.Init(config)
c.Assume(err, gs.IsNil)
startInput()
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
c.Expect(input.processMessageCount, gs.Equals, int64(0))
c.Expect(input.processMessageBytes, gs.Equals, int64(0))
})
})
}
func TestEndpointPinger(t *testing.T) {
useMockFuncs()
defer useStdFuncs()
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
gomock.Any(), gomock.Any()).AnyTimes()
mckStat := NewMockStatistician(mockCtrl)
mckPinger := NewMockPropPinger(mockCtrl)
mckStore := NewMockStore(mockCtrl)
mckWorker := NewMockWorker(mockCtrl)
Convey("Proprietary pings", t, func() {
app := NewApplication()
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
app.SetPropPinger(mckPinger)
app.SetStore(mckStore)
eh := NewEndpointHandler()
eh.setApp(app)
eh.setMaxDataLen(4096)
app.SetEndpointHandler(eh)
Convey("Should return early if the pinger can bypass the WebSocket", func() {
uaid := "91357e1a34714cadacb3f13cf47a2736"
app.AddWorker(uaid, mckWorker)
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{},
URL: &url.URL{Path: "/update/123"},
Body: nil,
}
gomock.InOrder(
mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
mckStat.EXPECT().Increment("updates.appserver.incoming"),
mckPinger.EXPECT().Send(uaid, int64(1257894000), "").Return(true, nil),
mckPinger.EXPECT().CanBypassWebsocket().Return(true),
mckStat.EXPECT().Increment("updates.appserver.received"),
mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
)
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 200)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, "{}")
})
Convey("Should continue if the pinger cannot bypass the WebSocket", func() {
uaid := "e3fc2cf1dc44424685010148b076d08b"
app.AddWorker(uaid, mckWorker)
data := randomText(eh.maxDataLen)
vals := make(url.Values)
vals.Set("data", data)
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{},
URL: &url.URL{Path: "/update/123"},
Body: formReader(vals),
}
gomock.InOrder(
mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
mckStat.EXPECT().Increment("updates.appserver.incoming"),
mckPinger.EXPECT().Send(uaid, int64(1257894000), data).Return(true, nil),
mckPinger.EXPECT().CanBypassWebsocket().Return(false),
mckStore.EXPECT().Update(uaid, "456", int64(1257894000)),
mckWorker.EXPECT().Send("456", int64(1257894000), data),
mckStat.EXPECT().Increment("updates.appserver.received"),
mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
)
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 200)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, "{}")
})
Convey("Should continue if the pinger fails", func() {
uaid := "8f412f5cb2384183bf60f7da26737271"
app.AddWorker(uaid, mckWorker)
vals := make(url.Values)
vals.Set("version", "7")
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{},
URL: &url.URL{Path: "/update/123"},
Body: formReader(vals),
}
gomock.InOrder(
mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
mckStat.EXPECT().Increment("updates.appserver.incoming"),
mckPinger.EXPECT().Send(uaid, int64(7), "").Return(
true, errors.New("oops")),
mckStore.EXPECT().Update(uaid, "456", int64(7)),
mckWorker.EXPECT().Send("456", int64(7), ""),
mckStat.EXPECT().Increment("updates.appserver.received"),
mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
)
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 200)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, "{}")
})
})
}
func TestSocketListenerConfig(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
gomock.Any(), gomock.Any()).AnyTimes()
mckStat := NewMockStatistician(mockCtrl)
tests := []struct {
name string
hostname string
conf listenerConfig
url string
ok bool
}{
{
name: "Invalid address, TLS, default hostname",
hostname: "example.com",
conf: listenerConfig{
listener: newMockListener(netAddr{"test", "!@#$"}),
useTLS: true,
maxConns: 5,
},
url: "wss://example.com",
ok: true,
},
{
name: "IPv6 address, default hostname",
hostname: "localhost",
conf: listenerConfig{
listener: newMockListener(netAddr{"test", "[::1]:8080"}),
useTLS: false,
maxConns: 1,
},
url: "ws://localhost:8080",
ok: true,
},
{
name: "IPv4 address, TLS, no default hostname",
conf: listenerConfig{
listener: newMockListener(netAddr{"test", "127.0.0.1:8090"}),
useTLS: true,
maxConns: 1,
},
url: "wss://127.0.0.1:8090",
ok: true,
},
}
for _, test := range tests {
func() {
app := NewApplication()
if len(test.hostname) > 0 {
app.hostname = test.hostname
}
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
sh := NewSocketHandler()
defer sh.Close()
sh.setApp(app)
err := sh.listenWithConfig(test.conf)
if err != nil {
if test.ok {
t.Errorf("On test %s, got listener error: %s", test.name, err)
}
return
}
if !test.ok {
listener := sh.Listener()
t.Errorf("On test %s, got %#v; want listener error", test.name, listener)
listener.Close()
return
}
if actual := sh.URL(); actual != test.url {
t.Errorf("Mismatched handler URL: got %q; want %q",
actual, test.url)
}
if actual := sh.MaxConns(); actual != test.conf.maxConns {
t.Errorf("Mismatched maximum connection count: got %d; want %d",
actual, test.conf.maxConns)
}
}()
}
}
func TestEndpointResolveKey(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
gomock.Any(), gomock.Any()).AnyTimes()
mckStat := NewMockStatistician(mockCtrl)
mckStore := NewMockStore(mockCtrl)
Convey("Endpoint tokens", t, func() {
app := NewApplication()
app.SetLogger(mckLogger)
app.SetMetrics(mckStat)
app.SetStore(mckStore)
Convey("Should return a 404 for invalid tokens", func() {
app.SetTokenKey("c3v0AlmmxXu_LSfdZY3l3eayLsIwkX48")
eh := NewEndpointHandler()
eh.setApp(app)
app.SetEndpointHandler(eh)
resp := httptest.NewRecorder()
req := &http.Request{
Method: "PUT",
Header: http.Header{},
URL: &url.URL{
Path: "/update/j1bqzFq9WiwFZbqay-y7xVlfSvtO1eY="}, // "123.456"
}
gomock.InOrder(
mckStore.EXPECT().KeyToIDs("123.456").Return("", "", ErrInvalidKey),
mckStat.EXPECT().Increment("updates.appserver.invalid"),
)
eh.ServeMux().ServeHTTP(resp, req)
So(resp.Code, ShouldEqual, 404)
body, isJSON := getJSON(resp.HeaderMap, resp.Body)
So(isJSON, ShouldBeTrue)
So(body.String(), ShouldEqual, `"Invalid Token"`)
})
Convey("Should not decode plaintext tokens without a key", func() {
var err error
app.SetTokenKey("")
eh := NewEndpointHandler()
eh.setApp(app)
app.SetEndpointHandler(eh)
_, err = eh.decodePK("")
So(err, ShouldNotBeNil)
pk, err := eh.decodePK("123.456")
So(pk, ShouldEqual, "123.456")
})
Convey("Should normalize decoded tokens", func() {
app.SetTokenKey("LM1xDImCx0rB46LCnx-3v4-Iyfk1LeKJbx9wuvx_z3U=")
eh := NewEndpointHandler()
eh.setApp(app)
app.SetEndpointHandler(eh)
// Hyphenated IDs should be normalized.
uaid := "dbda2ba2-004c-491f-9e3d-c5950aee93de"
chid := "848cd568-3f2a-4108-9ce4-bd0d928ecad4"
// " \t%s.%s\r\n" % (uaid, chid)
encodedKey := "qfGSdZzwf20GXiYubmZfIXj11Rx4RGJujFsjSQGdF4LRBhHbB_vt3hdW7cRvL9Fq_t_guMBGkDgebOoa5gRd1GGLN-Cv6h5hkpRTbdju8Tk-hMyC91BP4CEres_8"
// decodePK should trim whitespace from encoded keys.
mckStore.EXPECT().KeyToIDs(
fmt.Sprintf("%s.%s", uaid, chid)).Return(uaid, chid, nil)
actualUAID, actualCHID, err := eh.resolvePK(encodedKey)
So(err, ShouldBeNil)
So(actualUAID, ShouldEqual, uaid)
So(actualCHID, ShouldEqual, chid)
})
Convey("Should reject invalid tokens", func() {
var err error
app.SetTokenKey("IhnNwMNbsFWiafTXSgF4Ag==")
eh := NewEndpointHandler()
eh.setApp(app)
app.SetEndpointHandler(eh)
invalidKey := "b54QOw2omSWBiEq0IuyfBGxHBIR7AI9YhCMA0lP9" // "_=!@#$%^&*()[]"
uaid := "82398a648c834f8b838cb3945eceaf29"
chid := "af445ad07e5f46b7a6c858150fc5aa92"
validKey := fmt.Sprintf("%s.%s", uaid, chid)
encodedKey := "swKSH8P2qprRt5y0J4Wi7ybl-qzFv1j09WPOfuabpEJmVUqwUpxjprXc2R3Yw0ITbqc_Swntw9_EpCgo_XuRTn7Q7opQYoQUgMPhCgT0EGbK"
_, _, err = eh.resolvePK(invalidKey[:8])
So(err, ShouldNotBeNil)
_, _, err = eh.resolvePK(invalidKey)
So(err, ShouldNotBeNil)
// Reject plaintext tokens if a key is specified.
_, _, err = eh.resolvePK(validKey)
So(err, ShouldNotBeNil)
mckStore.EXPECT().KeyToIDs(validKey).Return("", "", ErrInvalidKey)
_, _, err = eh.resolvePK(encodedKey)
So(err, ShouldNotBeNil)
mckStore.EXPECT().KeyToIDs(validKey).Return(uaid, chid, nil)
actualUAID, actualCHID, err := eh.resolvePK(encodedKey)
So(err, ShouldBeNil)
So(actualUAID, ShouldEqual, uaid)
So(actualCHID, ShouldEqual, chid)
})
})
}
func MultiDecoderSpec(c gospec.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil) // initializes Globals()
c.Specify("A MultiDecoder", func() {
subsTOML := `[StartsWithM]
type = "PayloadRegexDecoder"
match_regex = '^(?P<TheData>m.*)'
log_errors = true
[StartsWithM.message_fields]
StartsWithM = "%TheData%"
[StartsWithS]
type = "PayloadRegexDecoder"
match_regex = '^(?P<TheData>s.*)'
log_errors = true
[StartsWithS.message_fields]
StartsWithS = "%TheData%"
[StartsWithM2]
type = "PayloadRegexDecoder"
match_regex = '^(?P<TheData>m.*)'
log_errors = true
[StartsWithM2.message_fields]
StartsWithM2 = "%TheData%"
`
RegisterPlugin("PayloadRegexDecoder", func() interface{} {
return &PayloadRegexDecoder{}
})
defer delete(AvailablePlugins, "PayloadRegexDecoder")
var configFile ConfigFile
_, err := toml.Decode(subsTOML, &configFile)
c.Assume(err, gs.IsNil)
decoder := new(MultiDecoder)
decoder.SetName("MyMultiDecoder")
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*MultiDecoderConfig)
supply := make(chan *PipelinePack, 1)
pack := NewPipelinePack(supply)
mSection, ok := configFile["StartsWithM"]
c.Assume(ok, gs.IsTrue)
mMaker, err := NewPluginMaker("StartsWithM", pConfig, mSection)
c.Assume(err, gs.IsNil)
pConfig.DecoderMakers["StartsWithM"] = mMaker
conf.Subs = []string{"StartsWithM"}
errMsg := "All subdecoders failed."
dRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// An error will be spit out b/c there's no real *dRunner in there;
// doesn't impact the tests.
dRunner.EXPECT().LogError(gomock.Any())
c.Specify("decodes simple messages", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
regex_data := "matching text"
pack.Message.SetPayload(regex_data)
_, err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
value, ok := pack.Message.GetFieldValue("StartsWithM")
c.Assume(ok, gs.IsTrue)
c.Expect(value, gs.Equals, regex_data)
})
c.Specify("returns an error if all decoders fail", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
regex_data := "non-matching text"
pack.Message.SetPayload(regex_data)
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, errMsg)
})
c.Specify("logs subdecoder failures when configured to do so", func() {
conf.LogSubErrors = true
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
regex_data := "non-matching text"
pack.Message.SetPayload(regex_data)
// Expect that we log an error for undecoded message.
dRunner.EXPECT().LogError(fmt.Errorf(
"Subdecoder 'StartsWithM' decode error: No match: %s", regex_data)).AnyTimes()
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, errMsg)
})
c.Specify("sets subdecoder runner correctly", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
// Call LogError to appease the angry gomock gods.
dRunner.LogError(errors.New("foo"))
// Now create a real *dRunner, pass it in, make sure a wrapper
// gets handed to the subdecoder.
dr := NewDecoderRunner(decoder.Name, decoder, 10)
decoder.SetDecoderRunner(dr)
sub := decoder.Decoders[0]
subRunner := sub.(*PayloadRegexDecoder).dRunner
c.Expect(subRunner.Name(), gs.Equals,
fmt.Sprintf("%s-StartsWithM", decoder.Name))
c.Expect(subRunner.Decoder(), gs.Equals, sub)
})
c.Specify("with multiple registered decoders", func() {
sSection, ok := configFile["StartsWithS"]
c.Assume(ok, gs.IsTrue)
sMaker, err := NewPluginMaker("StartsWithS", pConfig, sSection)
c.Assume(err, gs.IsNil)
pConfig.DecoderMakers["StartsWithS"] = sMaker
m2Section, ok := configFile["StartsWithM2"]
c.Assume(ok, gs.IsTrue)
m2Maker, err := NewPluginMaker("StartsWithM2", pConfig, m2Section)
c.Assume(err, gs.IsNil)
pConfig.DecoderMakers["StartsWithM2"] = m2Maker
conf.Subs = append(conf.Subs, "StartsWithS", "StartsWithM2")
// Two more subdecoders means two more LogError calls.
dRunner.EXPECT().LogError(gomock.Any()).Times(2)
c.Specify("defaults to `first-wins` cascading", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("on a first match condition", func() {
pack.Message.SetPayload("match first")
_, err = decoder.Decode(pack)
c.Expect(err, gs.IsNil)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsTrue)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
c.Specify("and a second match condition", func() {
pack.Message.SetPayload("second match")
_, err = decoder.Decode(pack)
c.Expect(err, gs.IsNil)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsTrue)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
c.Specify("returning an error if they all fail", func() {
pack.Message.SetPayload("won't match")
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, errMsg)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
})
c.Specify("and using `all` cascading", func() {
conf.CascadeStrategy = "all"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("matches multiples when appropriate", func() {
pack.Message.SetPayload("matches twice")
_, err = decoder.Decode(pack)
c.Expect(err, gs.IsNil)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsTrue)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsTrue)
})
c.Specify("matches singles when appropriate", func() {
pack.Message.SetPayload("second match")
_, err = decoder.Decode(pack)
c.Expect(err, gs.IsNil)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsTrue)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
c.Specify("returns an error if they all fail", func() {
pack.Message.SetPayload("won't match")
packs, err := decoder.Decode(pack)
c.Expect(len(packs), gs.Equals, 0)
c.Expect(err.Error(), gs.Equals, errMsg)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
})
})
})
c.Specify("A MultiDecoder w/ MultiOutput", func() {
subsTOML := `[sub0]
type = "MultiOutputDecoder"
[sub1]
type = "MultiOutputDecoder"
[sub2]
type = "MultiOutputDecoder"
`
RegisterPlugin("MultiOutputDecoder", func() interface{} {
return &MultiOutputDecoder{}
})
defer delete(AvailablePlugins, "MultiOutputDecoder")
var configFile ConfigFile
_, err := toml.Decode(subsTOML, &configFile)
decoder := new(MultiDecoder)
decoder.SetName("MyMultiDecoder")
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*MultiDecoderConfig)
conf.CascadeStrategy = "all"
supply := make(chan *PipelinePack, 10)
pack := NewPipelinePack(supply)
sub0Section, ok := configFile["sub0"]
c.Assume(ok, gs.IsTrue)
sub0Maker, err := NewPluginMaker("sub0", pConfig, sub0Section)
c.Assume(err, gs.IsNil)
pConfig.DecoderMakers["sub0"] = sub0Maker
sub1Section, ok := configFile["sub1"]
c.Assume(ok, gs.IsTrue)
sub1Maker, err := NewPluginMaker("sub1", pConfig, sub1Section)
c.Assume(err, gs.IsNil)
pConfig.DecoderMakers["sub1"] = sub1Maker
sub2Section, ok := configFile["sub2"]
c.Assume(ok, gs.IsTrue)
sub2Maker, err := NewPluginMaker("sub2", pConfig, sub2Section)
c.Assume(err, gs.IsNil)
pConfig.DecoderMakers["sub2"] = sub2Maker
conf.Subs = []string{"sub0", "sub1", "sub2"}
dRunner := pipelinemock.NewMockDecoderRunner(ctrl)
c.Specify("always tries to decode all packs", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
packs, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(len(packs), gs.Equals, 15)
})
})
}
func TestLocatorReadyNotify(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
uaid := "fce61180716a40ed8e79bf5ff0ba34bc"
mckLogger := NewMockLogger(mockCtrl)
mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(),
gomock.Any()).AnyTimes()
var (
// routerPipes maps fake peer addresses to their respective pipes. Used
// by dialRouter to connect to peers.
routerPipes = make(map[netAddr]*pipeListener)
// contacts is a list of peer URLs for the locator.
contacts []string
)
// Fake listener for the sender's router, used to test self-routing.
sndRouterAddr := netAddr{"tcp", "snd-router.example.com:3000"}
sndRouterPipe := newPipeListener()
defer sndRouterPipe.Close()
routerPipes[sndRouterAddr] = sndRouterPipe
contacts = append(contacts, "http://snd-router.example.com:3000")
// Fake listener for the receiver's router, used to test routing updates
// to different hosts.
recvRouterAddr := netAddr{"tcp", "recv-router.example.com:3000"}
recvRouterPipe := newPipeListener()
defer recvRouterPipe.Close()
routerPipes[recvRouterAddr] = recvRouterPipe
contacts = append(contacts, "http://recv-router.example.com:3000")
// Fake listener for the receiver's WebSocket handler, used to accept a
// WebSocket client connection.
socketHandlerPipe := newPipeListener()
defer socketHandlerPipe.Close()
// Fake locator.
mckLocator := NewMockLocator(mockCtrl)
mckLocator.EXPECT().Contacts(uaid).Return(contacts, nil).Times(2)
// Fake dialer to connect to each peer's routing listener.
dialRouter := func(network, address string) (net.Conn, error) {
if pipe, ok := routerPipes[netAddr{network, address}]; ok {
return pipe.Dial(network, address)
}
return nil, &netErr{temporary: false, timeout: false}
}
// Configures a fake router for the app.
setRouter := func(app *Application, listener net.Listener) {
r := NewBroadcastRouter()
r.setApp(app)
r.setClientOptions(10, 3*time.Second, 3*time.Second) // Defaults.
r.setClientTransport(&http.Transport{Dial: dialRouter})
r.listenWithConfig(listenerConfig{listener: listener})
r.maxDataLen = 4096
r.server = newServeWaiter(&http.Server{Handler: r.ServeMux()})
app.SetRouter(r)
}
// sndApp is the server broadcasting the update. The locator returns the
// addresses of the sender and receiver to test self-routing.
sndApp := NewApplication()
sndApp.SetLogger(mckLogger)
sndStat := NewMockStatistician(mockCtrl)
sndApp.SetMetrics(sndStat)
sndStore := NewMockStore(mockCtrl)
sndApp.SetStore(sndStore)
sndApp.SetLocator(mckLocator)
// Set up a fake router for the sender.
setRouter(sndApp, sndRouterPipe)
// recvApp is the server receiving the update.
recvApp := NewApplication()
recvApp.SetLogger(mckLogger)
recvStat := NewMockStatistician(mockCtrl)
recvApp.SetMetrics(recvStat)
recvStore := NewMockStore(mockCtrl)
recvApp.SetStore(recvStore)
// Wrap the fake locator in a type that implements ReadyNotifier.
recvLocator := newMockReadyNotifier(mckLocator)
recvApp.SetLocator(recvLocator)
// Set up a fake WebSocket handler for the receiver.
recvSocketHandler := NewSocketHandler()
recvSocketHandler.setApp(recvApp)
recvSocketHandler.listenWithConfig(listenerConfig{
listener: socketHandlerPipe})
recvSocketHandler.server = newServeWaiter(&http.Server{Handler: recvSocketHandler.ServeMux()})
recvApp.SetSocketHandler(recvSocketHandler)
// Set up a fake router for the receiver.
setRouter(recvApp, recvRouterPipe)
chid := "2b7c5c27d6224bfeaf1c158c3c57fca3"
version := int64(2)
data := "I'm a little teapot, short and stout."
var wg sync.WaitGroup // Waits for the client to close.
wg.Add(1)
dialChan := make(chan bool) // Signals when the client connects.
timeout := closeAfter(2 * time.Second)
go func() {
defer wg.Done()
origin := &url.URL{Scheme: "ws", Host: "recv-conn.example.com"}
ws, err := dialSocketListener(socketHandlerPipe, &websocket.Config{
Location: origin,
Origin: origin,
Version: websocket.ProtocolVersionHybi13,
})
if err != nil {
t.Errorf("Error dialing host: %s", err)
return
}
defer ws.Close()
err = websocket.JSON.Send(ws, struct {
Type string `json:"messageType"`
DeviceID string `json:"uaid"`
ChannelIDs []string `json:"channelIDs"`
}{"hello", uaid, []string{}})
if err != nil {
t.Errorf("Error writing handshake request: %s", err)
return
}
helloReply := new(HelloReply)
if err = websocket.JSON.Receive(ws, helloReply); err != nil {
t.Errorf("Error reading handshake reply: %s", err)
return
}
select {
case dialChan <- true:
case <-timeout:
t.Errorf("Timed out waiting for router")
return
}
flushReply := new(FlushReply)
if err = websocket.JSON.Receive(ws, flushReply); err != nil {
t.Errorf("Error reading routed update: %s", err)
return
}
ok := false
expected := Update{chid, uint64(version), data}
for _, update := range flushReply.Updates {
if ok = update == expected; ok {
break
}
}
if !ok {
t.Errorf("Missing update %#v in %#v", expected, flushReply.Updates)
return
}
}()
// Start the handlers.
errChan := make(chan error, 3)
go sndApp.Router().Start(errChan)
go recvApp.SocketHandler().Start(errChan)
go recvApp.Router().Start(errChan)
// First and second routing attempts to self.
sndStat.EXPECT().Increment("updates.routed.unknown").Times(2)
// Initial routing attempt to peer.
recvStat.EXPECT().Increment("updates.routed.unknown")
// Client connects to peer.
recvStat.EXPECT().Increment("client.socket.connect")
recvStore.EXPECT().CanStore(0).Return(true)
recvStat.EXPECT().Increment("updates.client.hello")
recvStore.EXPECT().FetchAll(uaid, gomock.Any()).Return(nil, nil, nil)
recvStat.EXPECT().Timer("client.flush", gomock.Any())
// Second routing attempt to peer.
recvStat.EXPECT().Increment("updates.routed.incoming")
recvStat.EXPECT().Increment("updates.sent")
recvStat.EXPECT().Timer("client.flush", gomock.Any())
recvStat.EXPECT().Increment("updates.routed.received")
recvStat.EXPECT().Timer("client.socket.lifespan", gomock.Any())
recvStat.EXPECT().Increment("client.socket.disconnect")
// Initial routing attempt should fail; the WebSocket listener shouldn't
// accept client connections before the locator is ready.
delivered, err := sndApp.Router().Route(nil, uaid, chid, version, timeNow(),
"disconnected", data)
if err != nil {
t.Errorf("Error routing to disconnected client: %s", err)
} else if delivered {
t.Error("Should not route to disconnected client")
}
// Signal the locator is ready, then wait for the client to connect.
recvLocator.SignalReady()
select {
case <-dialChan:
case <-time.After(5 * time.Second):
t.Fatalf("Timed out waiting for the client to connect")
}
// Routing should succeed once the client is connected.
delivered, err = sndApp.Router().Route(nil, uaid, chid, version, timeNow(),
"connected", data)
if err != nil {
t.Errorf("Error routing to connected client: %s", err)
} else if !delivered {
t.Error("Should route to connected client")
}
gomock.InOrder(
mckLocator.EXPECT().Close(),
recvStore.EXPECT().Close(),
)
if err := recvApp.Close(); err != nil {
t.Errorf("Error closing peer: %s", err)
}
wg.Wait()
gomock.InOrder(
mckLocator.EXPECT().Close(),
sndStore.EXPECT().Close(),
)
if err := sndApp.Close(); err != nil {
t.Errorf("Error closing self: %s", err)
}
// Wait for the handlers to stop.
for i := 0; i < 3; i++ {
<-errChan
}
}
func HttpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
var (
reqMethod string
reqBody string
reqHeader http.Header
handleWg sync.WaitGroup
runWg sync.WaitGroup
delay bool
)
handler := new(_test_handler)
handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) {
defer handleWg.Done()
if delay {
time.Sleep(time.Duration(2) * time.Millisecond)
}
// Capture request body for test assertions.
p := make([]byte, req.ContentLength)
_, err := req.Body.Read(p)
c.Expect(err.Error(), gs.Equals, "EOF")
reqBody = string(p)
// Capture request method and headers for test assertions.
reqMethod = req.Method
reqHeader = req.Header
// Respond with either a response body or a response code.
if len(handler.respBody) > 0 {
rw.Write([]byte(handler.respBody))
} else {
rw.WriteHeader(handler.respCode)
}
flusher := rw.(http.Flusher)
flusher.Flush()
}
oth := ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encConfig := new(plugins.PayloadEncoderConfig)
err := encoder.Init(encConfig)
c.Expect(err, gs.IsNil)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
c.Specify("An HttpOutput", func() {
httpOutput := new(HttpOutput)
config := httpOutput.ConfigStruct().(*HttpOutputConfig)
c.Specify("barfs on bogus URLs", func() {
config.Address = "one-two-three-four"
err := httpOutput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
server := httptest.NewServer(handler)
defer server.Close()
runOutput := func() {
httpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
runWg.Done()
}
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
payload := "this is the payload"
pack.Message.SetPayload(payload)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(
[]byte(payload), nil)
config.Address = server.URL
handler.respBody = "Response Body"
c.Specify("makes http POST requests by default", func() {
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(reqMethod, gs.Equals, "POST")
})
c.Specify("makes http PUT requests", func() {
config.Method = "put"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(reqMethod, gs.Equals, "PUT")
})
c.Specify("makes http GET requests", func() {
config.Method = "get"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, "")
c.Expect(reqMethod, gs.Equals, "GET")
})
c.Specify("correctly passes headers along", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"]))
c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"]))
})
c.Specify("uses http auth when specified", func() {
config.Username = "******"
config.Password = "******"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
auth := reqHeader.Get("Authorization")
c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue)
decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:])
c.Expect(err, gs.IsNil)
c.Expect(string(decodedAuth), gs.Equals, "user:pass")
})
c.Specify("logs error responses", func() {
handler.respBody = ""
handler.respCode = 500
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
var errMsg string
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
func(err error) {
errMsg = err.Error()
})
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(strings.HasPrefix(errMsg,
"HTTP Error code returned: 500"), gs.IsTrue)
})
c.Specify("honors http timeout interval", func() {
config.HttpTimeout = 1 // 1 millisecond
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
var errMsg string
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
func(err error) {
errMsg = err.Error()
})
delay = true
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(strings.Contains(errMsg, "use of closed network connection"),
gs.IsTrue)
})
})
})
}
func LogstreamerInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
here, _ := os.Getwd()
dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
c.Expect(tmpErr, gs.Equals, nil)
defer func() {
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
config := NewPipelineConfig(globals)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A LogstreamerInput", func() {
lsInput := &LogstreamerInput{pConfig: config}
lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
lsiConfig.LogDirectory = dirPath
lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
lsiConfig.Differentiator = []string{"logfile"}
lsiConfig.Priority = []string{"^Seq"}
lsiConfig.Decoder = "decoder-name"
c.Specify("w/ no translation map", func() {
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
c.Expect(len(lsInput.plugins), gs.Equals, 1)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 5 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
"-"+lsiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
runOutChan := make(chan error, 1)
go func() {
err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
runOutChan <- err
}()
d, _ := time.ParseDuration("5s")
timeout := time.After(d)
timed := false
for x := 0; x < numLines; x++ {
select {
case <-ith.DecodeChan:
case <-timeout:
timed = true
x += numLines
}
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lsInput.Stop()
c.Expect(timed, gs.Equals, false)
c.Expect(<-runOutChan, gs.Equals, nil)
})
})
c.Specify("with a translation map", func() {
lsiConfig.Translation = make(ls.SubmatchTranslationMap)
lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)
c.Specify("allows len 1 translation map for 'missing'", func() {
lsiConfig.Translation["Seq"]["missing"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
})
c.Specify("doesn't allow len 1 map for other keys", func() {
lsiConfig.Translation["Seq"]["missin"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"A translation map with one entry ('Seq') must be specifying a "+
"'missing' key.")
})
})
})
}
func TcpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
c.Specify("A TcpInput", func() {
tcpInput := &TcpInput{}
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
}
bytesChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var srDoneWG sync.WaitGroup
startServer := func() {
srDoneWG.Add(1)
ith.MockInputRunner.EXPECT().Name().Return("mock_name")
ith.MockInputRunner.EXPECT().NewDeliverer(gomock.Any()).Return(ith.MockDeliverer)
ith.MockDeliverer.EXPECT().Done()
ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
ith.MockSplitterRunner.EXPECT().Done().Do(func() {
srDoneWG.Done()
})
// splitCall gets called twice. The first time it returns nil, the
// second time it returns io.EOF.
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
ith.MockDeliverer).AnyTimes()
splitCall.Do(func(conn net.Conn, del Deliverer) {
recd, _ := ioutil.ReadAll(conn)
bytesChan <- recd
splitCall.Return(io.EOF)
})
err := tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}
c.Specify("not using TLS", func() {
err := tcpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(tcpInput.listener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
c.Specify("accepts connections and passes them to the splitter", func() {
go startServer()
data := []byte("THIS IS THE DATA")
outConn, err := net.Dial("tcp", ith.AddrStr)
c.Assume(err, gs.IsNil)
_, err = outConn.Write(data)
c.Expect(err, gs.IsNil)
outConn.Close()
recd := <-bytesChan
c.Expect(err, gs.IsNil)
c.Expect(string(recd), gs.Equals, string(data))
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
})
c.Specify("using TLS", func() {
config.UseTls = true
c.Specify("fails to init w/ missing key or cert file", func() {
config.Tls = TlsConfig{}
err := tcpInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"TLS config requires both cert_file and key_file value.")
})
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
}
c.Specify("accepts connections and passes them to the splitter", func() {
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go startServer()
data := []byte("This is a test.")
clientConfig := new(tls.Config)
clientConfig.InsecureSkipVerify = true
outConn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Assume(err, gs.IsNil)
outConn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
n, err := outConn.Write(data)
c.Expect(err, gs.IsNil)
c.Expect(n, gs.Equals, len(data))
outConn.Close()
recd := <-bytesChan
c.Expect(err, gs.IsNil)
c.Expect(string(recd), gs.Equals, string(data))
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
c.Specify("doesn't accept connections below specified min TLS version", func() {
config.Tls.MinVersion = "TLS12"
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go startServer()
clientConfig := &tls.Config{
InsecureSkipVerify: true,
MaxVersion: tls.VersionTLS11,
}
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(conn, gs.IsNil)
c.Expect(err, gs.Not(gs.IsNil))
<-bytesChan
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
})
})
}