func LogstreamerInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
here, _ := os.Getwd()
dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
c.Expect(tmpErr, gs.Equals, nil)
defer func() {
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
config := NewPipelineConfig(globals)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A LogstreamerInput", func() {
lsInput := &LogstreamerInput{pConfig: config}
lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
lsiConfig.LogDirectory = dirPath
lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
lsiConfig.Differentiator = []string{"logfile"}
lsiConfig.Priority = []string{"^Seq"}
lsiConfig.Decoder = "decoder-name"
c.Specify("w/ no translation map", func() {
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
c.Expect(len(lsInput.plugins), gs.Equals, 1)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 5 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
"-"+lsiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
runOutChan := make(chan error, 1)
go func() {
err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
runOutChan <- err
}()
d, _ := time.ParseDuration("5s")
timeout := time.After(d)
timed := false
for x := 0; x < numLines; x++ {
select {
case <-ith.DecodeChan:
case <-timeout:
timed = true
x += numLines
}
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lsInput.Stop()
c.Expect(timed, gs.Equals, false)
c.Expect(<-runOutChan, gs.Equals, nil)
})
})
c.Specify("with a translation map", func() {
lsiConfig.Translation = make(ls.SubmatchTranslationMap)
lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)
c.Specify("allows len 1 translation map for 'missing'", func() {
lsiConfig.Translation["Seq"]["missing"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
})
c.Specify("doesn't allow len 1 map for other keys", func() {
lsiConfig.Translation["Seq"]["missin"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"A translation map with one entry ('Seq') must be specifying a "+
"'missing' key.")
})
})
})
}
func UdpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr,
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("UdpInput")
encCall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "UdpInput-ProtobufDecoder")
encCall.Return(ith.Decoder, true)
c.Specify("reads a message from the connection and passes it to the decoder", func() {
hbytes, _ := proto.Marshal(header)
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
c.Assume(err, gs.IsNil)
buf := encodeMessage(hbytes, mbytes)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
udpInput.Stop()
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Decoded, gs.IsFalse)
})
})
c.Specify("A UdpInput Multiline input", func() {
ith.AddrStr = "localhost:55566"
ith.ResolvedAddrStr = "127.0.0.1:55566"
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr,
Decoder: "test",
ParserType: "token"})
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).Times(2)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(2)
ith.MockInputRunner.EXPECT().Name().Return("UdpInput").AnyTimes()
encCall := ith.MockHelper.EXPECT().DecoderRunner("test", "UdpInput-test")
encCall.Return(ith.Decoder, true)
c.Specify("reads two messages from a packet and passes them to the decoder", func() {
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
c.Assume(err, gs.IsNil)
_, err = conn.Write([]byte("message1\nmessage2\n"))
c.Assume(err, gs.IsNil)
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message1\n")
ith.PackSupply <- ith.Pack
packRef = <-ith.DecodeChan
c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message2\n")
udpInput.Stop()
})
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockDRunner := NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
c.Specify("with a valid setup and no decoder", func() {
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a message", func() {
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
})
c.Specify("with a valid setup using a decoder", func() {
decoderName := "defaultDecoder"
config.Decoder = decoderName
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
// Mock up our default decoder runner and decoder.
ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
decCall.Return(mockDRunner, true)
mockDecoder := NewMockDecoder(ctrl)
mockDRunner.EXPECT().Decoder().Return(mockDecoder)
c.Specify("consumes a message", func() {
packs := []*PipelinePack{ith.Pack}
mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
c.Specify("consumes a serialized message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the decoded pack
mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
// Ignore leading 5 bytes of encoded message as thats the header
c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close
ug.Add(1)
// Expect the close and the InChan calls
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func ProcessInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A ProcessInput", func() {
pInput := ProcessInput{}
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes()
ith.MockInputRunner.EXPECT().Name().Return("logger").AnyTimes()
enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "logger-RegexpDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).AnyTimes()
config := pInput.ConfigStruct().(*ProcessInputConfig)
config.Command = make(map[string]cmdConfig)
pConfig := NewPipelineConfig(nil)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
c.Specify("reads a message from ProcessInput", func() {
pInput.SetName("SimpleTest")
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
config.Delimiter = "|"
// Note that no working directory is explicitly specified
config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_TEST1_CMD, Args: PROCESSINPUT_TEST1_CMD_ARGS}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
expected_payloads := PROCESSINPUT_TEST1_OUTPUT
actual_payloads := []string{}
for x := 0; x < 4; x++ {
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
actual_payloads = append(actual_payloads, *packRef.Message.Payload)
fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")
// Free up the scheduler
runtime.Gosched()
}
for x := 0; x < 4; x++ {
c.Expect(expected_payloads[x], gs.Equals, actual_payloads[x])
}
pInput.Stop()
})
c.Specify("handles bad arguments", func() {
pInput.SetName("BadArgs")
config.ParseStdout = false
config.ParseStderr = true
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
config.Delimiter = "|"
// Note that no working directory is explicitly specified
config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
expected_err := fmt.Errorf("BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]")
ith.MockInputRunner.EXPECT().LogError(expected_err)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
ith.PackSupply <- ith.Pack
<-ith.DecodeChan
runtime.Gosched()
pInput.Stop()
})
c.Specify("can pipe multiple commands together", func() {
pInput.SetName("PipedCmd")
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
// Overload the delimiter
config.Delimiter = " "
// Note that no working directory is explicitly specified
config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD1, Args: PROCESSINPUT_PIPE_CMD1_ARGS}
config.Command["1"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD2, Args: PROCESSINPUT_PIPE_CMD2_ARGS}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
expected_payloads := PROCESSINPUT_PIPE_OUTPUT
actual_payloads := []string{}
for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ {
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
actual_payloads = append(actual_payloads, *packRef.Message.Payload)
fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")
// Free up the scheduler
runtime.Gosched()
}
for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ {
c.Expect(fmt.Sprintf("[%d] [%s] [%x]",
len(actual_payloads[x]),
actual_payloads[x],
actual_payloads[x]),
gs.Equals,
fmt.Sprintf("[%d] [%s] [%x]",
len(expected_payloads[x]),
expected_payloads[x],
expected_payloads[x]))
}
pInput.Stop()
})
})
}
func TcpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
key := "testkey"
signers := map[string]Signer{"test_1": {key}}
signer := "test"
c.Specify("A TcpInput protobuf parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
Signers: signers,
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr)
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
enccall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "TcpInput-123-ProtobufDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
cleanup := func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}
c.Specify("reads a message from its connection", func() {
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
})
c.Specify("reads a MD5 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a SHA1 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_SHA1)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(sha1.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a signed message with an expired key from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(11)) // non-existent key version
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write([]byte("some bytes"))
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
})
c.Specify("A TcpInput regexp parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
Decoder: "RegexpDecoder",
ParserType: "regexp",
}
tcpInput := TcpInput{}
err := tcpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
mbytes := []byte("this is a test message\n")
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("logger")
enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "TcpInput-123-RegexpDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
c.Specify("reads a message from its connection", func() {
readCall.Return(len(mbytes), nil)
readCall.Do(getPayloadText(mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes[:len(mbytes)-1]))
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
})
})
c.Specify("A TcpInput token parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
Decoder: "TokenDecoder",
ParserType: "token",
Delimiter: "\n"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
mbytes := []byte("this is a test message\n")
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("logger")
enccall := ith.MockHelper.EXPECT().DecoderRunner("TokenDecoder", "TcpInput-123-TokenDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
c.Specify("reads a message from its connection", func() {
readCall.Return(len(mbytes), nil)
readCall.Do(getPayloadText(mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
})
})
c.Specify("A TcpInput using TLS", func() {
tcpInput := TcpInput{}
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
ParserType: "token",
UseTls: true,
}
c.Specify("fails to init w/ missing key or cert file", func() {
config.Tls = TlsConfig{}
err := tcpInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("accepts TLS client connections", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
}
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
tcpInput.Stop()
tcpInput.wg.Wait()
}()
clientConfig := new(tls.Config)
clientConfig.InsecureSkipVerify = true
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(err, gs.IsNil)
defer conn.Close()
conn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
n, err := conn.Write([]byte("This is a test."))
c.Expect(err, gs.IsNil)
c.Expect(n, gs.Equals, len("This is a test."))
})
c.Specify("doesn't accept connections below specified min TLS version", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
MinVersion: "TLS12",
}
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
tcpInput.Stop()
tcpInput.wg.Wait()
time.Sleep(time.Duration(1000))
}()
clientConfig := &tls.Config{
InsecureSkipVerify: true,
MaxVersion: tls.VersionTLS11,
}
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(conn, gs.IsNil)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func ProcessDirectoryInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
//ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes()
ith.DecodeChan = make(chan *PipelinePack)
//mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
//mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).AnyTimes()
c.Specify("A ProcessDirectoryInput", func() {
pdiInput := ProcessDirectoryInput{}
//ith.MockInputRunner.EXPECT().Name().Return("logger").AnyTimes()
// Ignore all messages
//ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig)
workingDir, err := os.Getwd()
c.Assume(err, gs.IsNil)
config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes")
pConfig := NewPipelineConfig(nil)
// `Ticker` is the last thing called during the setup part of the
// input's `Run` method, so it triggers a waitgroup that tests can
// wait on when they need to ensure initialization has finished.
var started sync.WaitGroup
started.Add(1)
tickChan := make(chan time.Time, 1)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do(
func() {
started.Done()
})
// Similarly we use a waitgroup to signal when LogMessage has been
// called to know when reloads have completed. Warning: If you call
// expectLogMessage with a msg that is never passed to LogMessage and
// then you call loaded.Wait() then your test will hang and never
// complete.
var loaded sync.WaitGroup
expectLogMessage := func(msg string) {
loaded.Add(1)
ith.MockInputRunner.EXPECT().LogMessage(msg).Do(
func(msg string) {
loaded.Done()
})
}
// Same name => same content.
paths := []string{
filepath.Join(config.ProcessDir, "100", "h0.toml"),
filepath.Join(config.ProcessDir, "100", "h1.toml"),
filepath.Join(config.ProcessDir, "200", "h0.toml"),
filepath.Join(config.ProcessDir, "300", "h1.toml"),
}
copyFile := func(src, dest string) {
inFile, err := os.Open(src)
c.Assume(err, gs.IsNil)
outFile, err := os.Create(dest)
c.Assume(err, gs.IsNil)
_, err = io.Copy(outFile, inFile)
c.Assume(err, gs.IsNil)
}
err = pdiInput.Init(config)
c.Expect(err, gs.IsNil)
ith.MockHelper.EXPECT().PipelineConfig().Times(4).Return(pConfig)
for _, p := range paths {
expectLogMessage("Added: " + p)
}
go pdiInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
pdiInput.Stop()
for _, entry := range pdiInput.inputs {
entry.ir.Input().Stop()
}
}()
started.Wait()
c.Specify("loads scheduled jobs", func() {
pathIndex := func(name string) (i int) {
var p string
for i, p = range paths {
if name == p {
return
}
}
return -1
}
for name, entry := range pdiInput.inputs {
i := pathIndex(name)
// Make sure each file path got registered.
c.Expect(i, gs.Not(gs.Equals), -1)
dirName := filepath.Base(filepath.Dir(name))
dirInt, err := strconv.Atoi(dirName)
c.Expect(err, gs.IsNil)
// And that the ticker interval was read correctly.
c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval)
}
})
c.Specify("discovers and adds a new job", func() {
// Copy one of the files to register a new process.
newPath := filepath.Join(config.ProcessDir, "300", "h0.toml")
copyFile(paths[0], newPath)
defer func() {
err := os.Remove(newPath)
c.Assume(err, gs.IsNil)
}()
// Set up expectations and trigger process dir reload.
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
expectLogMessage("Added: " + newPath)
tickChan <- time.Now()
loaded.Wait()
// Make sure our plugin was loaded.
c.Expect(len(pdiInput.inputs), gs.Equals, 5)
newEntry, ok := pdiInput.inputs[newPath]
c.Expect(ok, gs.IsTrue)
c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300))
})
c.Specify("removes a deleted job", func() {
err := os.Remove(paths[3])
c.Assume(err, gs.IsNil)
defer func() {
copyFile(paths[1], paths[3])
}()
// Set up expectations and trigger process dir reload.
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
expectLogMessage("Removed: " + paths[3])
tickChan <- time.Now()
loaded.Wait()
// Make sure our plugin was deleted.
c.Expect(len(pdiInput.inputs), gs.Equals, 3)
})
c.Specify("notices a changed job", func() {
// Overwrite one job w/ a slightly different one.
copyFile(paths[0], paths[3])
defer copyFile(paths[1], paths[3])
// Set up expectations and trigger process dir reload.
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig).AnyTimes()
expectLogMessage("Removed: " + paths[3])
expectLogMessage("Added: " + paths[3])
tickChan <- time.Now()
loaded.Wait()
// Make sure the new config was loaded.
c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals,
"hello world\n")
})
})
}
func LogfileInputSpec0(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A LogFileInput", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "test-seekjournal"
lfiConfig.LogFile = "../testsupport/test-zeus.log"
lfiConfig.Logger = "zeus"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = "decoder-name"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 95 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lfInput.Stop()
fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile)
c.Expect(err, gs.IsNil)
fileStr := string(fileBytes)
lines := strings.Split(fileStr, "\n")
for i, line := range lines {
if line == "" {
continue
}
c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus")
}
// Wait for the file update to hit the disk; better suggestions are welcome
runtime.Gosched()
time.Sleep(time.Millisecond * 250)
journalData := []byte(`{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_len":300,"last_start":28650,"seek":28950}`)
journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
c.Expect(err, gs.IsNil)
c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
})
c.Specify("uses the filename as the default logger name", func() {
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.LogFile = "../testsupport/test-zeus.log"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.Equals, nil)
c.Expect(lfInput.Monitor.logger_ident,
gs.Equals,
lfiConfig.LogFile)
})
})
c.Specify("A Regex LogFileInput", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "regex-seekjournal"
lfiConfig.LogFile = "../testsupport/test-zeus.log"
lfiConfig.Logger = "zeus"
lfiConfig.ParserType = "regexp"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = "decoder-name"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 95 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("doesn't whack default RegexpParser delimiter", func() {
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
parser := lfInput.Monitor.parser.(*RegexpParser)
buf := bytes.NewBuffer([]byte("split\nhere"))
n, r, err := parser.Parse(buf)
c.Expect(n, gs.Equals, len("split\n"))
c.Expect(string(r), gs.Equals, "split")
c.Expect(err, gs.IsNil)
})
c.Specify("keeps captures in the record text", func() {
lfiConfig.Delimiter = "(\n)"
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
parser := lfInput.Monitor.parser.(*RegexpParser)
buf := bytes.NewBuffer([]byte("split\nhere"))
n, r, err := parser.Parse(buf)
c.Expect(n, gs.Equals, len("split\n"))
c.Expect(string(r), gs.Equals, "split\n")
c.Expect(err, gs.IsNil)
})
c.Specify("reads a log file", func() {
lfiConfig.Delimiter = "(\n)"
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lfInput.Stop()
fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile)
c.Expect(err, gs.IsNil)
fileStr := string(fileBytes)
lines := strings.Split(fileStr, "\n")
for i, line := range lines {
if line == "" {
continue
}
c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n")
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus")
}
// Wait for the file update to hit the disk; better suggestions are welcome
runtime.Gosched()
time.Sleep(time.Millisecond * 250)
journalData := []byte(`{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_len":300,"last_start":28650,"seek":28950}`)
journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
c.Expect(err, gs.IsNil)
c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
})
})
c.Specify("A Regex Multiline LogFileInput", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "multiline-seekjournal"
lfiConfig.LogFile = "../testsupport/multiline.log"
lfiConfig.Logger = "multiline"
lfiConfig.ParserType = "regexp"
lfiConfig.Delimiter = "\n(\\d{4}-\\d{2}-\\d{2})"
lfiConfig.DelimiterLocation = "start"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = "decoder-name"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 4 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lfInput.Stop()
lines := []string{
"2012-07-13 18:48:01 debug readSocket()",
"2012-07-13 18:48:21 info Processing queue id 3496 -> subm id 2817 from site ms",
"2012-07-13 18:48:25 debug line0\nline1\nline2",
"2012-07-13 18:48:26 debug readSocket()",
}
for i, line := range lines {
c.Expect(packs[i].Message.GetPayload(), gs.Equals, line)
c.Expect(packs[i].Message.GetLogger(), gs.Equals, "multiline")
}
// Wait for the file update to hit the disk; better suggestions are welcome
runtime.Gosched()
time.Sleep(time.Millisecond * 250)
journalData := []byte(`{"last_hash":"39e4c3e6e9c88a794b3e7c91c155682c34cf1a4a","last_len":41,"last_start":172,"seek":214}`)
journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
c.Expect(err, gs.IsNil)
c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
})
})
c.Specify("A message.proto LogFileInput", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "protobuf-seekjournal"
lfiConfig.LogFile = "../testsupport/protobuf.log"
lfiConfig.ParserType = "message.proto"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = "decoder-name"
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 7 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and decode outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lfInput.Stop()
lines := []int{36230, 41368, 42310, 41343, 37171, 56727, 46492}
for i, line := range lines {
c.Expect(len(packs[i].MsgBytes), gs.Equals, line)
err = proto.Unmarshal(packs[i].MsgBytes, packs[i].Message)
c.Expect(err, gs.IsNil)
c.Expect(packs[i].Message.GetType(), gs.Equals, "hekabench")
}
// Wait for the file update to hit the disk; better suggestions are welcome
runtime.Gosched()
time.Sleep(time.Millisecond * 250)
journalData := []byte(`{"last_hash":"f67dc6bbbbb6a91b59e661b6170de50c96eab100","last_len":46499,"last_start":255191,"seek":301690}`)
journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
c.Expect(err, gs.IsNil)
c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
})
})
c.Specify("A message.proto LogFileInput no decoder", func() {
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.SeekJournalName = "protobuf-seekjournal"
lfiConfig.LogFile = "../testsupport/protobuf.log"
lfiConfig.ParserType = "message.proto"
lfiConfig.UseSeekJournal = true
lfiConfig.Decoder = ""
lfiConfig.DiscoverInterval = 1
lfiConfig.StatInterval = 1
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.Not(gs.IsNil))
})
}
func LogfileInputSpec1(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
journalName := "test-seekjournal"
journalDir := filepath.Join(tmpDir, "seekjournal")
tmpErr = os.MkdirAll(journalDir, 0770)
c.Expect(tmpErr, gs.Equals, nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A LogfileInput", func() {
c.Specify("save the seek position of the last complete logline", func() {
lfInput, lfiConfig := createIncompleteLogfileInput(journalName)
// Initialize the input test helper
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
dName := "decoder-name"
lfInput.decoderName = dName
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 4 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
pbcall := ith.MockHelper.EXPECT().DecoderRunner(dName, "LogfileInput-"+dName)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
newFM := new(FileMonitor)
newFM.Init(lfiConfig)
c.Expect(err, gs.Equals, nil)
fbytes, _ := json.Marshal(lfInput.Monitor)
// Check that the persisted hashcode is from the last
// complete log line
expected_lastline := `10.1.1.4 plinko-565.byzantium.mozilla.com user3 [15/Mar/2013:12:20:27 -0700] "GET /1.1/user3/storage/passwords?newer=1356237662.44&full=1 HTTP/1.1" 200 1396 "-" "Firefox/20.0.1 FxSync/1.22.0.201304.desktop" "-" "ssl: SSL_RSA_WITH_RC4_128_SHA, version=TLSv1, bits=128" node_s:0.047167 req_s:0.047167 retries:0 req_b:446 "c_l:-"` + "\n"
c.Expect((strings.IndexAny(string(fbytes),
sha1_hexdigest(expected_lastline)) > -1), gs.IsTrue)
json.Unmarshal(fbytes, &newFM)
if runtime.GOOS == "windows" {
c.Expect(newFM.seek, gs.Equals, int64(1253))
} else {
c.Expect(newFM.seek, gs.Equals, int64(1249))
}
lfInput.Stop()
})
})
c.Specify("A LogfileDirectoryManagerInput", func() {
c.Specify("empty file name", func() {
var err error
ldm := new(LogfileDirectoryManagerInput)
conf := ldm.ConfigStruct().(*LogfileInputConfig)
conf.LogFile = ""
err = ldm.Init(conf)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "A logfile name must be specified.")
})
c.Specify("glob in file name", func() {
var err error
ldm := new(LogfileDirectoryManagerInput)
conf := ldm.ConfigStruct().(*LogfileInputConfig)
conf.LogFile = "../testsupport/*.log"
err = ldm.Init(conf)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "Globs are not allowed in the file name: *.log")
})
// Note: Testing the actual functionality (spinning up new plugins within Heka)
// is a manual process.
})
}
func FileMonitorSpec(c gs.Context) {
config := NewPipelineConfig(nil)
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
c.Expect(tmpErr, gs.Equals, nil)
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
journalName := "test-seekjournal"
journalDir := filepath.Join(tmpDir, "seekjournals")
tmpErr = os.MkdirAll(journalDir, 0770)
c.Expect(tmpErr, gs.Equals, nil)
journalPath := filepath.Join(journalDir, journalName)
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("saved last read position", func() {
c.Specify("without a previous journal", func() {
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
lfInput, lfiConfig := createLogfileInput(journalName)
// Initialize the input test helper
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
dName := "decoder-name"
lfInput.decoderName = dName
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 95 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
ith.MockInputRunner.EXPECT().Name().Return("FileMonitor")
ith.MockHelper.EXPECT().DecoderRunner(dName, "FileMonitor-"+dName).Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
go func() {
err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
}()
for x := 0; x < numLines; x++ {
_ = <-ith.DecodeChan
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
newFM := new(FileMonitor)
newFM.Init(lfiConfig)
c.Expect(err, gs.Equals, nil)
fbytes, _ := json.Marshal(lfInput.Monitor)
json.Unmarshal(fbytes, &newFM)
c.Expect(newFM.seek, gs.Equals, int64(28950))
lfInput.Stop()
})
c.Specify("with a previous journal initializes with a seek value", func() {
lfInput, lfiConfig := createLogfileInput(journalName)
journalData := `{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_start":28650,"last_len":300,"seek":28950}`
journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660)
c.Expect(journalErr, gs.Equals, nil)
journal.WriteString(journalData)
journal.Close()
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
c.Expect(err, gs.IsNil)
// # bytes should be set to what's in the journal data
c.Expect(lfInput.Monitor.seek, gs.Equals, int64(28950))
})
c.Specify("resets last read position to 0 if hash doesn't match", func() {
lfInput, lfiConfig := createLogfileInput(journalName)
lfiConfig.ResumeFromStart = true
journalData := `{"last_hash":"xxxxx","last_start":28650,"last_len":300,"seek":28950}`
journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660)
c.Expect(journalErr, gs.Equals, nil)
journal.WriteString(journalData)
journal.Close()
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
// # bytes should be set to what's in the journal data
c.Expect(lfInput.Monitor.seek, gs.Equals, int64(0))
})
c.Specify("resets last read position to end of file if hash doesn't match", func() {
lfInput, lfiConfig := createLogfileInput(journalName)
lfiConfig.ResumeFromStart = false
journalData := `{"last_hash":"xxxxx","last_start":28650,"last_len":300,"seek":28950}`
journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660)
c.Expect(journalErr, gs.Equals, nil)
journal.WriteString(journalData)
journal.Close()
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.IsNil)
// # bytes should be set to what's in the journal data
c.Expect(lfInput.Monitor.seek, gs.Equals, int64(28950))
})
})
c.Specify("filemonitor generates expected journal path", func() {
lfInput := new(LogfileInput)
lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
lfiConfig.LogFile = filepath.Join("..", "testsupport", "test-zeus.log")
lfiConfig.DiscoverInterval = 5
lfiConfig.StatInterval = 5
err := lfInput.Init(lfiConfig)
c.Expect(err, gs.Equals, nil)
clean := filepath.Join(journalDir, "___testsupport_test-zeus_log")
c.Expect(lfInput.Monitor.seekJournalPath, gs.Equals, clean)
})
}