func StatsdInputSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(InputTestHelper)
ith.Msg = getTestMessage()
ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
ith.PackSupply = make(chan *PipelinePack, 1)
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, input runner, and stat accumulator
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockStatAccum := NewMockStatAccumulator(ctrl)
c.Specify("A StatsdInput", func() {
statsdInput := StatsdInput{}
config := statsdInput.ConfigStruct().(*StatsdInputConfig)
config.Address = ith.AddrStr
err := statsdInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := statsdInput.listener
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockListener := ts.NewMockConn(ctrl)
statsdInput.listener = mockListener
ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil)
mockListener.EXPECT().Close()
mockListener.EXPECT().SetReadDeadline(gomock.Any())
c.Specify("sends a Stat to the StatAccumulator", func() {
statName := "sample.count"
statVal := 303
msg := fmt.Sprintf("%s:%d|c\n", statName, statVal)
expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)}
mockStatAccum.EXPECT().DropStat(expected).Return(true)
readCall := mockListener.EXPECT().Read(make([]byte, 512))
readCall.Return(len(msg), nil)
readCall.Do(func(msgBytes []byte) {
copy(msgBytes, []byte(msg))
statsdInput.Stop()
})
var wg sync.WaitGroup
wg.Add(1)
go func() {
err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
wg.Wait()
})
})
}
func InputsSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
msg := getTestMessage()
pipelinePack := getTestPipelinePack()
// Specify localhost, but we're not really going to use the network
addrStr := "localhost:55565"
resolvedAddrStr := "127.0.0.1:55565"
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{addrStr})
c.Assume(err, gs.IsNil)
realListener := (udpInput.Listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, resolvedAddrStr)
realListener.Close()
// Replace the listener object w/ a mock listener
mockListener := ts.NewMockConn(ctrl)
udpInput.Listener = mockListener
msgJson, _ := json.Marshal(msg)
putMsgJsonInBytes := func(msgBytes []byte) {
copy(msgBytes, msgJson)
}
c.Specify("reads a message from its listener", func() {
mockListener.EXPECT().SetReadDeadline(gomock.Any())
readCall := mockListener.EXPECT().Read(pipelinePack.MsgBytes)
readCall.Return(len(msgJson), nil)
readCall.Do(putMsgJsonInBytes)
second := time.Second
err := udpInput.Read(pipelinePack, &second)
c.Expect(err, gs.IsNil)
c.Expect(pipelinePack.Decoded, gs.IsFalse)
c.Expect(string(pipelinePack.MsgBytes), gs.Equals, string(msgJson))
})
})
}
func OutputsSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
tmpFileName)
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := getTestMessage()
pack := NewPipelinePack(pConfig.inputRecycleChan)
pack.Message = msg
pack.Decoded = true
toString := func(outData interface{}) string {
return string(*(outData.(*[]byte)))
}
c.Specify("correctly formats text output", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 20)
c.Specify("by default", func() {
fileOutput.handleMessage(pack, &outData)
c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
})
c.Specify("w/ a prepended timestamp when specified", func() {
fileOutput.prefix_ts = true
fileOutput.handleMessage(pack, &outData)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
payload := *msg.Payload
c.Expect(strContents, ts.StringContains, payload)
c.Expect(strContents, ts.StringStartsWith, todayStr)
})
})
c.Specify("correctly formats JSON output", func() {
config.Format = "json"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified", func() {
fileOutput.handleMessage(pack, &outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
})
c.Specify("and with a timestamp", func() {
fileOutput.prefix_ts = true
fileOutput.handleMessage(pack, &outData)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(strContents, ts.StringContains, string(msgJson)+"\n")
c.Expect(strContents, ts.StringStartsWith, todayStr)
})
})
c.Specify("correctly formats protocol buffer stream output", func() {
config.Format = "protobufstream"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified and timestamp ignored", func() {
fileOutput.prefix_ts = true
err := fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = tmpdir
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), ts.StringContains, "-------")
}
})
})
})
c.Specify("A TcpOutput", func() {
tcpOutput := new(TcpOutput)
config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
tcpOutput.connection = ts.NewMockConn(ctrl)
msg := getTestMessage()
pack := NewPipelinePack(pConfig.inputRecycleChan)
pack.Message = msg
pack.Decoded = true
c.Specify("correctly formats protocol buffer stream output", func() {
outBytes := make([]byte, 0, 200)
err := createProtobufStream(pack, &outBytes)
c.Expect(err, gs.IsNil)
b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
c.Expect(bytes.Equal(b, (outBytes)[:len(b)]), gs.IsTrue)
})
c.Specify("writes out to the network", func() {
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
}
ch <- "ready"
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch // wait for server
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
go func() {
wg.Add(1)
tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait() // wait for close to finish, prevents intermittent test failures
matchBytes := make([]byte, 0, 1000)
err = createProtobufStream(pack, &matchBytes)
c.Expect(err, gs.IsNil)
result = <-ch
c.Expect(result, gs.Equals, string(matchBytes))
})
})
c.Specify("Runner restarts a plugin on the first time only", func() {
pc := new(PipelineConfig)
var pluginGlobals PluginGlobals
pluginGlobals.Retries = RetryOptions{
MaxDelay: "1us",
Delay: "1us",
MaxJitter: "1us",
MaxRetries: 1,
}
pw := &PluginWrapper{
name: "stoppingOutput",
configCreator: func() interface{} { return nil },
pluginCreator: func() interface{} { return new(StoppingOutput) },
}
output := new(StoppingOutput)
pc.outputWrappers = make(map[string]*PluginWrapper)
pc.outputWrappers["stoppingOutput"] = pw
oRunner := NewFORunner("stoppingOutput", output, &pluginGlobals)
var wg sync.WaitGroup
cfgCall := oth.MockHelper.EXPECT().PipelineConfig()
cfgCall.Return(pc)
wg.Add(1)
oRunner.Start(oth.MockHelper, &wg) // no panic => success
wg.Wait()
c.Expect(stopoutputTimes, gs.Equals, 2)
})
c.Specify("Runner restarts plugin and resumes feeding it", func() {
pc := new(PipelineConfig)
var pluginGlobals PluginGlobals
pluginGlobals.Retries = RetryOptions{
MaxDelay: "1us",
Delay: "1us",
MaxJitter: "1us",
MaxRetries: 4,
}
pw := &PluginWrapper{
name: "stoppingresumeOutput",
configCreator: func() interface{} { return nil },
pluginCreator: func() interface{} { return new(StopResumeOutput) },
}
output := new(StopResumeOutput)
pc.outputWrappers = make(map[string]*PluginWrapper)
pc.outputWrappers["stoppingresumeOutput"] = pw
oRunner := NewFORunner("stoppingresumeOutput", output, &pluginGlobals)
var wg sync.WaitGroup
cfgCall := oth.MockHelper.EXPECT().PipelineConfig()
cfgCall.Return(pc)
wg.Add(1)
oRunner.Start(oth.MockHelper, &wg) // no panic => success
wg.Wait()
c.Expect(stopresumerunTimes, gs.Equals, 3)
c.Expect(len(stopresumeHolder), gs.Equals, 2)
c.Expect(stopresumeHolder[1], gs.Equals, "woot")
c.Expect(oRunner.retainPack, gs.IsNil)
})
}
func InputsSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(InputTestHelper)
ith.Msg = getTestMessage()
ith.Pack = NewPipelinePack(config.inputRecycleChan)
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.Decoders = make([]DecoderRunner, int(message.Header_JSON+1))
ith.Decoders[message.Header_PROTOCOL_BUFFER] = NewMockDecoderRunner(ctrl)
ith.Decoders[message.Header_JSON] = NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockDecoderSet = NewMockDecoderSet(ctrl)
key := "testkey"
signers := map[string]Signer{"test_1": {key}}
signer := "test"
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{ith.AddrStr, signers})
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
// replace the listener object w/ a mock listener
mockListener := ts.NewMockConn(ctrl)
udpInput.listener = mockListener
mbytes, _ := json.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageEncoding(message.Header_JSON)
header.SetMessageLength(uint32(len(mbytes)))
buf := make([]byte, message.MAX_MESSAGE_SIZE+message.MAX_HEADER_SIZE+3)
readCall := mockListener.EXPECT().Read(buf)
mockDecoderRunner := ith.Decoders[message.Header_JSON].(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Times(2).Return(ith.PackSupply)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
c.Specify("reads a message from the connection and passes it to the decoder", func() {
encCall := ith.MockDecoderSet.EXPECT().ByEncoding(message.Header_JSON)
encCall.Return(mockDecoderRunner, true)
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Decoded, gs.IsFalse)
})
c.Specify("reads a MD5 signed message from its connection", func() {
encCall := ith.MockDecoderSet.EXPECT().ByEncoding(message.Header_JSON)
encCall.Return(mockDecoderRunner, true)
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
})
c.Specify("invalid header", func() {
hbytes, _ := proto.Marshal(header)
hbytes[0] = 0
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
})
c.Specify("A TcpInput", func() {
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{ith.AddrStr, signers})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := ts.NewMockConn(ctrl)
mockListener := ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
buf := make([]byte, message.MAX_MESSAGE_SIZE+message.MAX_HEADER_SIZE+3)
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(buf)
neterr := ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoders[message.Header_PROTOCOL_BUFFER].(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
//ith.MockInputRunner.EXPECT().DecoderSource().Return(ith.MockDecoderSource)
//ith.MockDecoderSource.EXPECT().NewDecodersByEncoding().Return(ith.Decoders)
c.Specify("reads a message from its connection", func() {
pbcall := ith.MockDecoderSet.EXPECT().ByEncoding(message.Header_PROTOCOL_BUFFER)
pbcall.Return(mockDecoderRunner, true)
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
})
c.Specify("reads a MD5 signed message from its connection", func() {
pbcall := ith.MockDecoderSet.EXPECT().ByEncoding(message.Header_PROTOCOL_BUFFER)
pbcall.Return(mockDecoderRunner, true)
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a SHA1 signed message from its connection", func() {
pbcall := ith.MockDecoderSet.EXPECT().ByEncoding(message.Header_PROTOCOL_BUFFER)
pbcall.Return(mockDecoderRunner, true)
header.SetHmacHashFunction(message.Header_SHA1)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(sha1.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a signed message with an expired key from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(11)) // non-existent key version
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write([]byte("some bytes"))
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
})
c.Specify("Runner recovers from panic in input's `Run()` method", func() {
input := new(PanicInput)
iRunner := NewInputRunner("panic", input)
var wg sync.WaitGroup
cfgCall := ith.MockHelper.EXPECT().PipelineConfig()
cfgCall.Return(config)
wg.Add(1)
iRunner.Start(ith.MockHelper, &wg) // no panic => success
wg.Wait()
})
}
func InputsSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(InputTestHelper)
ith.Msg = getTestMessage()
ith.Pack = NewPipelinePack(config.inputRecycleChan)
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.Decoder = NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockDecoderSet = NewMockDecoderSet(ctrl)
key := "testkey"
signers := map[string]Signer{"test_1": {key}}
signer := "test"
c.Specify("A ProcessInput", func() {
pInput := ProcessInput{}
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes()
ith.MockInputRunner.EXPECT().Name().Return("logger").AnyTimes()
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet).AnyTimes()
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet).AnyTimes()
enccall := ith.MockDecoderSet.EXPECT().ByName("RegexpDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
mockDecoderRunner := ith.Decoder.(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).AnyTimes()
config := pInput.ConfigStruct().(*ProcessInputConfig)
config.Command = make(map[string]cmd_config)
pConfig := NewPipelineConfig(nil)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
c.Specify("reads a message from ProcessInput", func() {
config.Name = "SimpleTest"
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
config.Delimiter = "|"
// Note that no working directory is explicitly specified
config.Command["0"] = cmd_config{Bin: "/bin/cat", Args: []string{"../testsupport/process_input_test.txt"}}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
expected_payloads := []string{"this|", "is|", "a|", "test|"}
actual_payloads := []string{}
for x := 0; x < 4; x++ {
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
actual_payloads = append(actual_payloads, *packRef.Message.Payload)
fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")
// Free up the scheduler
runtime.Gosched()
}
for x := 0; x < 4; x++ {
c.Expect(expected_payloads[x], gs.Equals, actual_payloads[x])
}
pInput.Stop()
})
c.Specify("handles bad arguments", func() {
config.Name = "BadArgs"
config.ParseStdout = false
config.ParseStderr = true
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
config.Delimiter = "|"
// Note that no working directory is explicitly specified
config.Command["0"] = cmd_config{Bin: "/bin/cat", Args: []string{"../testsupport/not_a_file.txt"}}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
expected_err := fmt.Errorf("BadArgs CommandChain::Wait() error: [exit status 1]")
ith.MockInputRunner.EXPECT().LogError(expected_err)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
ith.PackSupply <- ith.Pack
<-ith.DecodeChan
runtime.Gosched()
pInput.Stop()
})
c.Specify("can pipe multiple commands together", func() {
config.Name = "PipedCmd"
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
// Overload the delimiter
config.Delimiter = " "
// Note that no working directory is explicitly specified
config.Command["0"] = cmd_config{Bin: "cat", Args: []string{"../testsupport/process_input_pipes_test.txt"}}
config.Command["1"] = cmd_config{Bin: "grep", Args: []string{"ignore"}}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
expected_payloads := []string{"ignore ", "this ", "line"}
actual_payloads := []string{}
for x := 0; x < 3; x++ {
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
actual_payloads = append(actual_payloads, *packRef.Message.Payload)
fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")
// Free up the scheduler
runtime.Gosched()
}
for x := 0; x < 3; x++ {
c.Expect(actual_payloads[x], gs.Equals, expected_payloads[x])
}
pInput.Stop()
})
})
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
err := udpInput.Init(&NetworkInputConfig{Address: ith.AddrStr,
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
mockDecoderRunner := ith.Decoder.(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
encCall := ith.MockDecoderSet.EXPECT().ByName("ProtobufDecoder")
encCall.Return(ith.Decoder, true)
c.Specify("reads a message from the connection and passes it to the decoder", func() {
hbytes, _ := proto.Marshal(header)
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
c.Assume(err, gs.IsNil)
buf := encodeMessage(hbytes, mbytes)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
udpInput.Stop()
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Decoded, gs.IsFalse)
})
})
c.Specify("A TcpInput protobuf parser", func() {
tcpInput := TcpInput{}
err := tcpInput.Init(&NetworkInputConfig{Address: ith.AddrStr,
Signers: signers,
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := ts.NewMockConn(ctrl)
mockListener := ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
enccall := ith.MockDecoderSet.EXPECT().ByName("ProtobufDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
c.Specify("reads a message from its connection", func() {
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
})
c.Specify("reads a MD5 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a SHA1 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_SHA1)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(sha1.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a signed message with an expired key from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(11)) // non-existent key version
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write([]byte("some bytes"))
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
})
c.Specify("A TcpInput regexp parser", func() {
tcpInput := TcpInput{}
err := tcpInput.Init(&NetworkInputConfig{Address: ith.AddrStr,
Decoder: "RegexpDecoder",
ParserType: "regexp",
Delimiter: "\n"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := ts.NewMockConn(ctrl)
mockListener := ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr)
mbytes := []byte("this is a test message\n")
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("logger")
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
enccall := ith.MockDecoderSet.EXPECT().ByName("RegexpDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
c.Specify("reads a message from its connection", func() {
readCall.Return(len(mbytes), nil)
readCall.Do(getPayloadText(mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes[:len(mbytes)-1]))
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
})
})
c.Specify("A TcpInput token parser", func() {
tcpInput := TcpInput{}
err := tcpInput.Init(&NetworkInputConfig{Address: ith.AddrStr,
Decoder: "TokenDecoder",
ParserType: "token",
Delimiter: "\n"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := ts.NewMockConn(ctrl)
mockListener := ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr)
mbytes := []byte("this is a test message\n")
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("logger")
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
enccall := ith.MockDecoderSet.EXPECT().ByName("TokenDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
c.Specify("reads a message from its connection", func() {
readCall.Return(len(mbytes), nil)
readCall.Do(getPayloadText(mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
})
})
c.Specify("Runner restarts a plugin on the first time only", func() {
var pluginGlobals PluginGlobals
pluginGlobals.Retries = RetryOptions{
MaxDelay: "1us",
Delay: "1us",
MaxJitter: "1us",
MaxRetries: 1,
}
pc := new(PipelineConfig)
pc.inputWrappers = make(map[string]*PluginWrapper)
pw := &PluginWrapper{
name: "stopping",
configCreator: func() interface{} { return nil },
pluginCreator: func() interface{} { return new(StoppingInput) },
}
pc.inputWrappers["stopping"] = pw
input := new(StoppingInput)
iRunner := NewInputRunner("stopping", input, &pluginGlobals)
var wg sync.WaitGroup
cfgCall := ith.MockHelper.EXPECT().PipelineConfig().Times(7)
cfgCall.Return(pc)
wg.Add(1)
iRunner.Start(ith.MockHelper, &wg)
wg.Wait()
c.Expect(stopinputTimes, gs.Equals, 2)
})
c.Specify("A LogFileInput", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "test-seekjournal"
lfiConfig.LogFile = "../testsupport/test-zeus.log"
lfiConfig.Logger = "zeus"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = "decoder-name"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
mockDecoderRunner := NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 95 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
pbcall := ith.MockDecoderSet.EXPECT().ByName(lfiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
close(lfInput.Monitor.stopChan)
fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile)
c.Expect(err, gs.IsNil)
fileStr := string(fileBytes)
lines := strings.Split(fileStr, "\n")
for i, line := range lines {
if line == "" {
continue
}
c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus")
}
// Wait for the file update to hit the disk; better suggestions are welcome
runtime.Gosched()
time.Sleep(time.Millisecond * 250)
journalData := []byte(`{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_len":300,"last_start":28650,"seek":28950}`)
journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
c.Expect(err, gs.IsNil)
c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
})
c.Specify("uses the filename as the default logger name", func() {
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.LogFile = "../testsupport/test-zeus.log"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.Equals, nil)
c.Expect(lfInput.Monitor.logger_ident,
gs.Equals,
lfiConfig.LogFile)
})
})
c.Specify("A Regex LogFileInput", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "regex-seekjournal"
lfiConfig.LogFile = "../testsupport/test-zeus.log"
lfiConfig.Logger = "zeus"
lfiConfig.ParserType = "regexp"
lfiConfig.Delimiter = "(\n)"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = "decoder-name"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
mockDecoderRunner := NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 95 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
pbcall := ith.MockDecoderSet.EXPECT().ByName(lfiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
close(lfInput.Monitor.stopChan)
fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile)
c.Expect(err, gs.IsNil)
fileStr := string(fileBytes)
lines := strings.Split(fileStr, "\n")
for i, line := range lines {
if line == "" {
continue
}
c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus")
}
// Wait for the file update to hit the disk; better suggestions are welcome
runtime.Gosched()
time.Sleep(time.Millisecond * 250)
journalData := []byte(`{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_len":300,"last_start":28650,"seek":28950}`)
journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
c.Expect(err, gs.IsNil)
c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
})
})
c.Specify("A Regex Multiline LogFileInput", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "multiline-seekjournal"
lfiConfig.LogFile = "../testsupport/multiline.log"
lfiConfig.Logger = "multiline"
lfiConfig.ParserType = "regexp"
lfiConfig.Delimiter = "\n(\\d{4}-\\d{2}-\\d{2})"
lfiConfig.DelimiterLocation = "start"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = "decoder-name"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
mockDecoderRunner := NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 4 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
pbcall := ith.MockDecoderSet.EXPECT().ByName(lfiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
close(lfInput.Monitor.stopChan)
lines := []string{
"2012-07-13 18:48:01 debug readSocket()",
"2012-07-13 18:48:21 info Processing queue id 3496 -> subm id 2817 from site ms",
"2012-07-13 18:48:25 debug line0\nline1\nline2",
"2012-07-13 18:48:26 debug readSocket()",
}
for i, line := range lines {
c.Expect(packs[i].Message.GetPayload(), gs.Equals, line)
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "multiline")
}
// Wait for the file update to hit the disk; better suggestions are welcome
runtime.Gosched()
time.Sleep(time.Millisecond * 250)
journalData := []byte(`{"last_hash":"39e4c3e6e9c88a794b3e7c91c155682c34cf1a4a","last_len":41,"last_start":172,"seek":214}`)
journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
c.Expect(err, gs.IsNil)
c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
})
})
c.Specify("A message.proto LogFileInput", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "protobuf-seekjournal"
lfiConfig.LogFile = "../testsupport/protobuf.log"
lfiConfig.ParserType = "message.proto"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = "decoder-name"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
mockDecoderRunner := NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 7 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and decode outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
pbcall := ith.MockDecoderSet.EXPECT().ByName(lfiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
close(lfInput.Monitor.stopChan)
lines := []int{36230, 41368, 42310, 41343, 37171, 56727, 46492}
for i, line := range lines {
c.Expect(len(packs[i].MsgBytes), gs.Equals, line)
err = proto.Unmarshal(packs[i].MsgBytes, packs[i].Message)
c.Expect(err, gs.IsNil)
c.Expect(packs[i].Message.GetType(), gs.Equals, "hekabench")
}
// Wait for the file update to hit the disk; better suggestions are welcome
runtime.Gosched()
time.Sleep(time.Millisecond * 250)
journalData := []byte(`{"last_hash":"f67dc6bbbbb6a91b59e661b6170de50c96eab100","last_len":46499,"last_start":255191,"seek":301690}`)
journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
c.Expect(err, gs.IsNil)
c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
})
})
c.Specify("A message.proto LogFileInput no decoder", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "protobuf-seekjournal"
lfiConfig.LogFile = "../testsupport/protobuf.log"
lfiConfig.ParserType = "message.proto"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = ""
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.Not(gs.IsNil))
})
}
func InputsSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(InputTestHelper)
ith.Msg = getTestMessage()
ith.Pack = NewPipelinePack(config.inputRecycleChan)
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.Decoders = make([]DecoderRunner, int(message.Header_JSON+1))
ith.Decoders[message.Header_PROTOCOL_BUFFER] = NewMockDecoderRunner(ctrl)
ith.Decoders[message.Header_JSON] = NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockDecoderSet = NewMockDecoderSet(ctrl)
key := "testkey"
signers := map[string]Signer{"test_1": {key}}
signer := "test"
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{ith.AddrStr, signers})
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
// replace the listener object w/ a mock listener
mockListener := ts.NewMockConn(ctrl)
udpInput.listener = mockListener
mbytes, _ := json.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageEncoding(message.Header_JSON)
header.SetMessageLength(uint32(len(mbytes)))
buf := make([]byte, message.MAX_MESSAGE_SIZE+message.MAX_HEADER_SIZE+3)
readCall := mockListener.EXPECT().Read(buf)
mockDecoderRunner := ith.Decoders[message.Header_JSON].(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Times(2).Return(ith.PackSupply)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
encCall := ith.MockDecoderSet.EXPECT().ByEncodings()
encCall.Return(ith.Decoders, nil)
c.Specify("reads a message from the connection and passes it to the decoder", func() {
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Decoded, gs.IsFalse)
})
c.Specify("reads a MD5 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
})
c.Specify("invalid header", func() {
hbytes, _ := proto.Marshal(header)
hbytes[0] = 0
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
})
c.Specify("A TcpInput", func() {
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{ith.AddrStr, signers})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := ts.NewMockConn(ctrl)
mockListener := ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
buf := make([]byte, message.MAX_MESSAGE_SIZE+message.MAX_HEADER_SIZE+3)
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(buf)
neterr := ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoders[message.Header_PROTOCOL_BUFFER].(*MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
enccall := ith.MockDecoderSet.EXPECT().ByEncodings()
enccall.Return(ith.Decoders, nil)
c.Specify("reads a message from its connection", func() {
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
})
c.Specify("reads a MD5 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a SHA1 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_SHA1)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(sha1.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a signed message with an expired key from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(11)) // non-existent key version
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write([]byte("some bytes"))
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, err)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go func() {
tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
ith.PackSupply <- ith.Pack
timeout := make(chan bool)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
})
c.Specify("Runner restarts a plugin on the first time only", func() {
var pluginGlobals PluginGlobals
pluginGlobals.Retries = RetryOptions{
MaxDelay: "1us",
Delay: "1us",
MaxJitter: "1us",
MaxRetries: 1,
}
pc := new(PipelineConfig)
pc.inputWrappers = make(map[string]*PluginWrapper)
pw := &PluginWrapper{
name: "stopping",
configCreator: func() interface{} { return nil },
pluginCreator: func() interface{} { return new(StoppingInput) },
}
pc.inputWrappers["stopping"] = pw
input := new(StoppingInput)
iRunner := NewInputRunner("stopping", input, &pluginGlobals)
var wg sync.WaitGroup
cfgCall := ith.MockHelper.EXPECT().PipelineConfig().Times(3)
cfgCall.Return(pc)
wg.Add(1)
iRunner.Start(ith.MockHelper, &wg)
wg.Wait()
c.Expect(stopinputTimes, gs.Equals, 2)
})
c.Specify("Runner recovers from panic in input's `Run()` method", func() {
input := new(PanicInput)
iRunner := NewInputRunner("panic", input, nil)
var wg sync.WaitGroup
cfgCall := ith.MockHelper.EXPECT().PipelineConfig()
cfgCall.Return(config)
wg.Add(1)
iRunner.Start(ith.MockHelper, &wg) // no panic => success
wg.Wait()
})
c.Specify("A LogFileInput", func() {
var err error
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournal, err = createJournal()
c.Expect(err, gs.IsNil)
lfiConfig.LogFile = "../testsupport/test-zeus.log"
lfiConfig.Logger = "zeus"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err = lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
dName := "decoder-name"
lfInput.decoderNames = []string{dName}
mockDecoderRunner := NewMockDecoderRunner(ctrl)
mockDecoder := NewMockDecoder(ctrl)
// Create pool of packs.
numLines := 95 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Inject(gomock.Any()).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)
pbcall := ith.MockDecoderSet.EXPECT().ByName(dName)
pbcall.Return(mockDecoderRunner, true)
mockDecoderRunner.EXPECT().Decoder().Return(mockDecoder)
decodeCall := mockDecoder.EXPECT().Decode(gomock.Any()).Times(numLines)
decodeCall.Return(nil)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for len(ith.PackSupply) > 0 {
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile)
c.Expect(err, gs.IsNil)
fileStr := string(fileBytes)
lines := strings.Split(fileStr, "\n")
for i, line := range lines {
if line == "" {
continue
}
c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus")
}
close(lfInput.Monitor.stopChan)
})
c.Specify("uses the filename as the default logger name", func() {
var err error
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournal, err = createJournal()
c.Expect(err, gs.Equals, nil)
lfiConfig.LogFile = "../testsupport/test-zeus.log"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err = lfInput.Init(lfiConfig)
c.Expect(err, gs.Equals, nil)
c.Expect(lfInput.Monitor.logger_ident,
gs.Equals,
lfiConfig.LogFile)
})
})
}