func BenchmarkMultiDecodeProtobuf(b *testing.B) {
b.StopTimer()
pConfig := NewPipelineConfig(nil) // initializes Globals
msg := pipeline_ts.GetTestMessage()
msg.SetPayload("This is a test")
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.MsgBytes, _ = proto.Marshal(msg)
decoder := new(MultiDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*MultiDecoderConfig)
sub := new(ProtobufDecoder)
sub.SetPipelineConfig(pConfig)
sub.Init(nil)
wrapper0 := NewPluginWrapper("sub", pConfig)
wrapper0.CreateWithError = func() (interface{}, error) {
return sub, nil
}
pConfig.DecoderWrappers["sub"] = wrapper0
conf.CascadeStrategy = "first-wins"
conf.Subs = []string{"sub"}
decoder.Init(conf)
b.StartTimer()
for i := 0; i < b.N; i++ {
decoder.Decode(pack)
}
}
func ProtobufDecoderSpec(c gospec.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
msg := ts.GetTestMessage()
config := NewPipelineConfig(nil) // Initializes globals.
c.Specify("A ProtobufDecoder", func() {
encoded, err := proto.Marshal(msg)
c.Assume(err, gs.IsNil)
pack := NewPipelinePack(config.inputRecycleChan)
decoder := new(ProtobufDecoder)
decoder.sampleDenominator = 1000 // Since we don't call decoder.Init().
c.Specify("decodes a protobuf message", func() {
pack.MsgBytes = encoded
_, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(pack.Message, gs.Equals, msg)
v, ok := pack.Message.GetFieldValue("foo")
c.Expect(ok, gs.IsTrue)
c.Expect(v, gs.Equals, "bar")
})
c.Specify("returns an error for bunk encoding", func() {
bunk := append([]byte{0, 0, 0}, encoded...)
pack.MsgBytes = bunk
_, err := decoder.Decode(pack)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func BenchmarkMultiDecodeProtobuf(b *testing.B) {
b.StopTimer()
pConfig := NewPipelineConfig(nil) // initializes Globals
msg := pipeline_ts.GetTestMessage()
msg.SetPayload("This is a test")
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.MsgBytes, _ = proto.Marshal(msg)
decoder := new(MultiDecoder)
decoder.SetPipelineConfig(pConfig)
conf := decoder.ConfigStruct().(*MultiDecoderConfig)
RegisterPlugin("ProtobufDecoder", func() interface{} {
return &ProtobufDecoder{}
})
defer delete(AvailablePlugins, "ProtobufDecoder")
var section PluginConfig
_, err := toml.Decode("", §ion)
if err != nil {
b.Fatalf("Error decoding empty TOML: %s", err.Error())
}
maker, err := NewPluginMaker("ProtobufDecoder", pConfig, section)
if err != nil {
b.Fatalf("Error decoding empty TOML: %s", err.Error())
}
pConfig.DecoderMakers["ProtobufDecoder"] = maker
conf.CascadeStrategy = "first-wins"
conf.Subs = []string{"sub"}
decoder.Init(conf)
b.StartTimer()
for i := 0; i < b.N; i++ {
decoder.Decode(pack)
}
}
func FilterRunnerSpec(c gs.Context) {
c.Specify("A filterrunner", func() {
pConfig := NewPipelineConfig(nil)
filter := &CounterFilter{}
commonFO := CommonFOConfig{
Matcher: "Type == 'bogus'",
}
chanSize := 10
fRunner, err := NewFORunner("counterFilter", filter, commonFO, "CounterFilter",
chanSize)
fRunner.h = pConfig
c.Assume(err, gs.IsNil)
pack := NewPipelinePack(pConfig.injectRecycleChan)
pConfig.injectRecycleChan <- pack
pack.Message = ts.GetTestMessage()
c.Assume(pack.TrustMsgBytes, gs.IsFalse)
msgEncoding, err := proto.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Specify("puts protobuf encoding into MsgBytes before delivery", func() {
result := fRunner.Inject(pack)
c.Expect(result, gs.IsTrue)
recd := <-pConfig.router.inChan
c.Expect(recd, gs.Equals, pack)
c.Expect(recd.TrustMsgBytes, gs.IsTrue)
c.Expect(bytes.Equal(msgEncoding, recd.MsgBytes), gs.IsTrue)
})
})
}
func StatsdInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.PackSupply = make(chan *PipelinePack, 1)
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, input runner, and stat accumulator
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockStatAccum := NewMockStatAccumulator(ctrl)
c.Specify("A StatsdInput", func() {
statsdInput := StatsdInput{}
config := statsdInput.ConfigStruct().(*StatsdInputConfig)
config.Address = ith.AddrStr
err := statsdInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := statsdInput.listener
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockListener := pipeline_ts.NewMockConn(ctrl)
statsdInput.listener = mockListener
ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil)
mockListener.EXPECT().Close()
mockListener.EXPECT().SetReadDeadline(gomock.Any())
c.Specify("sends a Stat to the StatAccumulator", func() {
statName := "sample.count"
statVal := 303
msg := fmt.Sprintf("%s:%d|c\n", statName, statVal)
expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)}
mockStatAccum.EXPECT().DropStat(expected).Return(true)
readCall := mockListener.EXPECT().Read(make([]byte, 512))
readCall.Return(len(msg), nil)
readCall.Do(func(msgBytes []byte) {
copy(msgBytes, []byte(msg))
statsdInput.Stop()
})
var wg sync.WaitGroup
wg.Add(1)
go func() {
err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
wg.Wait()
})
})
}
func BenchmarkEncodeProtobuf(b *testing.B) {
b.StopTimer()
msg := ts.GetTestMessage()
b.StartTimer()
for i := 0; i < b.N; i++ {
proto.Marshal(msg)
}
}
func MessageTemplateSpec(c gs.Context) {
c.Specify("A message template", func() {
mt := make(MessageTemplate)
mt["Logger"] = "test logger"
mt["Payload"] = "test payload"
mt["Hostname"] = "host.example.com"
mt["Pid"] = "123.456"
mt["Type"] = "test type"
mt["Severity"] = "23"
mt["tmplTest|baz"] = "bar"
msg := ts.GetTestMessage()
c.Specify("replaces message values", func() {
err := mt.PopulateMessage(msg, nil)
c.Assume(err, gs.IsNil)
c.Expect(msg.GetLogger(), gs.Equals, mt["Logger"])
c.Expect(msg.GetPayload(), gs.Equals, mt["Payload"])
c.Expect(msg.GetHostname(), gs.Equals, mt["Hostname"])
c.Expect(msg.GetPid(), gs.Equals, int32(123))
c.Expect(msg.GetType(), gs.Equals, mt["Type"])
c.Expect(msg.GetSeverity(), gs.Equals, int32(23))
fields := msg.FindAllFields("tmplTest")
c.Expect(len(fields), gs.Equals, 1)
field := fields[0]
value := field.GetValueString()
c.Expect(len(value), gs.Equals, 1)
c.Expect(value[0], gs.Equals, "bar")
c.Expect(field.GetRepresentation(), gs.Equals, "baz")
})
c.Specify("honors substitutions", func() {
mt["Payload"] = "this is %substitution%"
mt["Hostname"] = "%host%.example.com"
mt["Type"] = "another %substitution%"
mt["tmplTest|baz"] = "%fieldvalue%"
subs := make(map[string]string)
subs["host"] = "otherhost"
subs["substitution"] = "a test"
subs["fieldvalue"] = "wakajawaka"
err := mt.PopulateMessage(msg, subs)
c.Assume(err, gs.IsNil)
c.Expect(msg.GetLogger(), gs.Equals, mt["Logger"])
c.Expect(msg.GetPayload(), gs.Equals, "this is a test")
c.Expect(msg.GetHostname(), gs.Equals, "otherhost.example.com")
c.Expect(msg.GetPid(), gs.Equals, int32(123))
c.Expect(msg.GetType(), gs.Equals, "another a test")
c.Expect(msg.GetSeverity(), gs.Equals, int32(23))
fields := msg.FindAllFields("tmplTest")
c.Expect(len(fields), gs.Equals, 1)
field := fields[0]
value := field.GetValueString()
c.Expect(len(value), gs.Equals, 1)
c.Expect(value[0], gs.Equals, subs["fieldvalue"])
c.Expect(field.GetRepresentation(), gs.Equals, "baz")
})
})
}
func BenchmarkDecodeProtobuf(b *testing.B) {
b.StopTimer()
msg := ts.GetTestMessage()
encoded, _ := proto.Marshal(msg)
config := NewPipelineConfig(nil)
pack := NewPipelinePack(config.inputRecycleChan)
decoder := new(ProtobufDecoder)
pack.MsgBytes = encoded
b.StartTimer()
for i := 0; i < b.N; i++ {
decoder.Decode(pack)
}
}
func TcpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
key := "testkey"
signers := map[string]Signer{"test_1": {key}}
signer := "test"
c.Specify("A TcpInput protobuf parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
Signers: signers,
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr)
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
enccall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "TcpInput-123-ProtobufDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
cleanup := func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}
c.Specify("reads a message from its connection", func() {
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
})
c.Specify("reads a MD5 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a SHA1 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_SHA1)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(sha1.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a signed message with an expired key from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(11)) // non-existent key version
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write([]byte("some bytes"))
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
})
c.Specify("A TcpInput regexp parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
Decoder: "RegexpDecoder",
ParserType: "regexp",
}
tcpInput := TcpInput{}
err := tcpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
mbytes := []byte("this is a test message\n")
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("logger")
enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "TcpInput-123-RegexpDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
c.Specify("reads a message from its connection", func() {
readCall.Return(len(mbytes), nil)
readCall.Do(getPayloadText(mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes[:len(mbytes)-1]))
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
})
})
c.Specify("A TcpInput token parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
Decoder: "TokenDecoder",
ParserType: "token",
Delimiter: "\n"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
mbytes := []byte("this is a test message\n")
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("logger")
enccall := ith.MockHelper.EXPECT().DecoderRunner("TokenDecoder", "TcpInput-123-TokenDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
c.Specify("reads a message from its connection", func() {
readCall.Return(len(mbytes), nil)
readCall.Do(getPayloadText(mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
})
})
c.Specify("A TcpInput using TLS", func() {
tcpInput := TcpInput{}
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
ParserType: "token",
UseTls: true,
}
c.Specify("fails to init w/ missing key or cert file", func() {
config.Tls = TlsConfig{}
err := tcpInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("accepts TLS client connections", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
}
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
tcpInput.Stop()
tcpInput.wg.Wait()
}()
clientConfig := new(tls.Config)
clientConfig.InsecureSkipVerify = true
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(err, gs.IsNil)
defer conn.Close()
conn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
n, err := conn.Write([]byte("This is a test."))
c.Expect(err, gs.IsNil)
c.Expect(n, gs.Equals, len("This is a test."))
})
c.Specify("doesn't accept connections below specified min TLS version", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
MinVersion: "TLS12",
}
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
tcpInput.Stop()
tcpInput.wg.Wait()
time.Sleep(time.Duration(1000))
}()
clientConfig := &tls.Config{
InsecureSkipVerify: true,
MaxVersion: tls.VersionTLS11,
}
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(conn, gs.IsNil)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func DashboardOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
dashboardOutput := new(DashboardOutput)
dashboardOutput.pConfig = pConfig
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)
errChan := make(chan error, 1)
startOutput := func() {
go func() {
errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
}()
}
if runtime.GOOS != "windows" {
c.Specify("A DashboardOutput", func() {
tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
c.Assume(err, gs.IsNil)
config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
config.WorkingDirectory = tmpdir
c.Specify("Init halts if basedirectory is not writable", func() {
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
err = dashboardOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("that is running", func() {
startedChan := make(chan bool, 1)
defer close(startedChan)
ts := httptest.NewUnstartedServer(nil)
dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
ts.Start()
startedChan <- true
return nil
}
ticker := make(chan time.Time)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)
err := os.MkdirAll(tmpdir, 0700)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Noop
})
c.Specify("sets custom http headers", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err = dashboardOutput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = dashboardOutput.server
startOutput()
inChan <- pack
<-startedChan
resp, err := http.Get(ts.URL)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
// Verify headers are there
eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
c.Expect(eq, gs.IsTrue)
eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
c.Expect(eq, gs.IsTrue)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
ts.Close()
})
})
}
}
func LogstreamerInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
here, _ := os.Getwd()
dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
c.Expect(tmpErr, gs.Equals, nil)
defer func() {
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
pConfig := NewPipelineConfig(globals)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// Specify localhost, but we're not really going to use the network.
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// Set up mock helper, runner, and pack supply channel.
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
c.Specify("A LogstreamerInput", func() {
lsInput := &LogstreamerInput{pConfig: pConfig}
lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
lsiConfig.LogDirectory = dirPath
lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
lsiConfig.Differentiator = []string{"logfile"}
lsiConfig.Priority = []string{"^Seq"}
c.Specify("w/ no translation map", func() {
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
c.Expect(len(lsInput.plugins), gs.Equals, 1)
// Create pool of packs.
numLines := 5 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs.
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
gomock.Any()).Times(numLines)
line := "boo hoo foo foo"
getRecCall.Return(len(line), []byte(line), nil)
getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
getRecCall.Return(0, make([]byte, 0), io.EOF)
deliverChan := make(chan []byte, 1)
deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
ith.MockDeliverer).Times(numLines)
deliverCall.Do(func(record []byte, del Deliverer) {
deliverChan <- record
})
ith.MockDeliverer.EXPECT().Done()
runOutChan := make(chan error, 1)
go func() {
err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
runOutChan <- err
}()
dur, _ := time.ParseDuration("5s")
timeout := time.After(dur)
timed := false
for x := 0; x < numLines; x++ {
select {
case record := <-deliverChan:
c.Expect(string(record), gs.Equals, line)
case <-timeout:
timed = true
x += numLines
}
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lsInput.Stop()
c.Expect(timed, gs.Equals, false)
c.Expect(<-runOutChan, gs.Equals, nil)
})
})
c.Specify("with a translation map", func() {
lsiConfig.Translation = make(ls.SubmatchTranslationMap)
lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)
c.Specify("allows len 1 translation map for 'missing'", func() {
lsiConfig.Translation["Seq"]["missing"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
})
c.Specify("doesn't allow len 1 map for other keys", func() {
lsiConfig.Translation["Seq"]["missin"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"A translation map with one entry ('Seq') must be specifying a "+
"'missing' key.")
})
})
})
}
func ProcessInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A ProcessInput", func() {
pInput := ProcessInput{}
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes()
ith.MockInputRunner.EXPECT().Name().Return("logger").AnyTimes()
enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "logger-RegexpDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).AnyTimes()
config := pInput.ConfigStruct().(*ProcessInputConfig)
config.Command = make(map[string]cmdConfig)
pConfig := NewPipelineConfig(nil)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
c.Specify("reads a message from ProcessInput", func() {
pInput.SetName("SimpleTest")
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
config.Delimiter = "|"
// Note that no working directory is explicitly specified
config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_TEST1_CMD, Args: PROCESSINPUT_TEST1_CMD_ARGS}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
expected_payloads := PROCESSINPUT_TEST1_OUTPUT
actual_payloads := []string{}
for x := 0; x < 4; x++ {
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
actual_payloads = append(actual_payloads, *packRef.Message.Payload)
fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")
// Free up the scheduler
runtime.Gosched()
}
for x := 0; x < 4; x++ {
c.Expect(expected_payloads[x], gs.Equals, actual_payloads[x])
}
pInput.Stop()
})
c.Specify("handles bad arguments", func() {
pInput.SetName("BadArgs")
config.ParseStdout = false
config.ParseStderr = true
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
config.Delimiter = "|"
// Note that no working directory is explicitly specified
config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
expected_err := fmt.Errorf("BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]")
ith.MockInputRunner.EXPECT().LogError(expected_err)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
ith.PackSupply <- ith.Pack
<-ith.DecodeChan
runtime.Gosched()
pInput.Stop()
})
c.Specify("can pipe multiple commands together", func() {
pInput.SetName("PipedCmd")
config.Decoder = "RegexpDecoder"
config.ParserType = "token"
// Overload the delimiter
config.Delimiter = " "
// Note that no working directory is explicitly specified
config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD1, Args: PROCESSINPUT_PIPE_CMD1_ARGS}
config.Command["1"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD2, Args: PROCESSINPUT_PIPE_CMD2_ARGS}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
expected_payloads := PROCESSINPUT_PIPE_OUTPUT
actual_payloads := []string{}
for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ {
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
actual_payloads = append(actual_payloads, *packRef.Message.Payload)
fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")
// Free up the scheduler
runtime.Gosched()
}
for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ {
c.Expect(fmt.Sprintf("[%d] [%s] [%x]",
len(actual_payloads[x]),
actual_payloads[x],
actual_payloads[x]),
gs.Equals,
fmt.Sprintf("[%d] [%s] [%x]",
len(expected_payloads[x]),
expected_payloads[x],
expected_payloads[x]))
}
pInput.Stop()
})
})
}
func SmtpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
c.Specify("A SmtpOutput", func() {
smtpOutput := new(SmtpOutput)
config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
config.SendTo = []string{"root"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
runnerName.Return("SmtpOutput")
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()
c.Specify("send email payload message", func() {
err := smtpOutput.Init(config)
c.Assume(err, gs.IsNil)
smtpOutput.sendFunction = testSendMail
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
go func() {
wg.Add(1)
smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait()
})
})
// Use this test with a real server
// c.Specify("Real SmtpOutput output", func() {
// smtpOutput := new(SmtpOutput)
//
// config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
// config.SendTo = []string{"root"}
//
// msg := pipeline_ts.GetTestMessage()
// pack := NewPipelinePack(pConfig.InputRecycleChan())
// pack.Message = msg
// pack.Decoded = true
// inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
// inChanCall.Return(inChan)
// runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
// runnerName.Return("SmtpOutput")
// oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()
//
// c.Specify("send a real email essage", func() {
//
// err := smtpOutput.Init(config)
// c.Assume(err, gs.IsNil)
//
// outStr := "Write me out to the network"
// pack.Message.SetPayload(outStr)
// go func() {
// wg.Add(1)
// smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
// wg.Done()
// }()
// inChan <- pack
// time.Sleep(1000) // allow time for the message output
// close(inChan)
// wg.Wait()
// // manually check the mail
// })
// })
}
func UdpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
udpOutput := new(UdpOutput)
config := udpOutput.ConfigStruct().(*UdpOutputConfig)
oth := plugins_ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encoder.Init(new(plugins.PayloadEncoderConfig))
inChan := make(chan *pipeline.PipelinePack, 1)
rChan := make(chan *pipeline.PipelinePack, 1)
var wg sync.WaitGroup
var rAddr net.Addr
c.Specify("A UdpOutput", func() {
msg := pipeline_ts.GetTestMessage()
payload := "Write me out to the network."
msg.SetPayload(payload)
pack := pipeline.NewPipelinePack(rChan)
pack.Message = msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
c.Specify("using UDP", func() {
addr := "127.0.0.1:45678"
config.Address = addr
ch := make(chan string, 1)
collectData := func() {
conn, err := net.ListenPacket("udp", addr)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = "localhost:12345"
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "udp")
c.Expect(rAddr.String(), gs.Equals, "127.0.0.1:12345")
close(inChan)
wg.Wait()
})
})
c.Specify("using Unix datagrams", func() {
if runtime.GOOS == "windows" {
return
}
testUnixAddr := func() string {
f, err := ioutil.TempFile("", "_heka_test_sock")
c.Assume(err, gs.IsNil)
addr := f.Name()
f.Close()
os.Remove(addr)
return addr
}
config.Address = testUnixAddr()
config.Net = "unixgram"
ch := make(chan string, 1)
var wg sync.WaitGroup
var rAddr net.Addr
collectData := func() {
conn, err := net.ListenPacket("unixgram", config.Address)
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
b := make([]byte, 1000)
var n int
n, rAddr, _ = conn.ReadFrom(b)
ch <- string(b[:n])
conn.Close()
err = os.Remove(config.Address)
var errMsg string
if err != nil {
errMsg = err.Error()
}
ch <- errMsg
}
go collectData()
result := <-ch // Wait for server to be ready.
c.Assume(result, gs.Equals, "ready")
c.Specify("writes out to the network", func() {
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
close(inChan)
wg.Wait()
result = <-ch
c.Expect(result, gs.Equals, "")
})
c.Specify("uses the specified local address", func() {
config.LocalAddress = testUnixAddr()
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- pack
result = <-ch
c.Expect(result, gs.Equals, payload)
c.Expect(rAddr.Network(), gs.Equals, "unixgram")
c.Expect(rAddr.String(), gs.Equals, config.LocalAddress)
close(inChan)
wg.Wait()
})
})
})
c.Specify("drop message contents if their size is bigger than allowed UDP datagram size", func() {
huge_msg := pipeline_ts.GetTestMessage()
payload := strings.Repeat("2", 131014)
huge_msg.SetPayload(payload)
huge_pack := pipeline.NewPipelinePack(rChan)
huge_pack.Message = huge_msg
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(huge_pack).Return(encoder.Encode(huge_pack))
oth.MockOutputRunner.EXPECT().LogError(fmt.Errorf("Message has exceeded allowed UDP data size: 131014 > 65507"))
config.Address = "localhost:12345"
err := udpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
inChan <- huge_pack
close(inChan)
wg.Wait()
})
c.Specify("checks validation of of Maximum message size limit", func() {
config.Address = "localhost:12345"
config.MaxMessageSize = 100
err := udpOutput.Init(config)
c.Assume(err.Error(), gs.Equals, "Maximum message size can't be smaller than 512 bytes.")
})
}
func CarbonOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := NewCarbonTestHelper(ctrl)
var wg sync.WaitGroup
pConfig := NewPipelineConfig(nil)
// make test data
const count = 5
lines := make([]string, count)
baseTime := time.Now().UTC().Add(-10 * time.Second)
for i := 0; i < count; i++ {
statName := fmt.Sprintf("stats.name.%d", i)
statTime := baseTime.Add(time.Duration(i) * time.Second)
lines[i] = fmt.Sprintf("%s %d %d", statName, i*2, statTime.Unix())
}
submit_data := strings.Join(lines, "\n")
expected_data := submit_data + "\n"
// a helper to make new test packs
newpack := func() *PipelinePack {
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
pack.Message.SetPayload(submit_data)
return pack
}
// collectDataTCP and collectDataUDP functions; when ready reports its port on
// chPort, or error on chErr; when data is received it is reported on chData
collectDataTCP := func(chPort chan<- int, chData chan<- string, chError chan<- error) {
tcpaddr, err := net.ResolveTCPAddr("tcp", ":0")
if err != nil {
chError <- err
return
}
listener, err := net.ListenTCP("tcp", tcpaddr)
if err != nil {
chError <- err
return
}
chPort <- listener.Addr().(*net.TCPAddr).Port
conn, err := listener.Accept()
if err != nil {
chError <- err
return
}
b := make([]byte, 10000)
n, err := conn.Read(b)
if err != nil {
chError <- err
return
}
chData <- string(b[0:n])
}
collectDataUDP := func(chPort chan<- int, chData chan<- string, chError chan<- error) {
udpaddr, err := net.ResolveUDPAddr("udp", ":0")
if err != nil {
chError <- err
return
}
conn, err := net.ListenUDP("udp", udpaddr)
if err != nil {
chError <- err
return
}
chPort <- conn.LocalAddr().(*net.UDPAddr).Port
b := make([]byte, 10000)
n, _, err := conn.ReadFromUDP(b)
if err != nil {
chError <- err
return
}
chData <- string(b[0:n])
}
doit := func(protocol string, collectData collectFunc) {
c.Specify("A CarbonOutput ", func() {
inChan := make(chan *PipelinePack, 1)
carbonOutput := new(CarbonOutput)
config := carbonOutput.ConfigStruct().(*CarbonOutputConfig)
pack := newpack()
c.Specify("writes "+protocol+" to the network", func() {
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
chError := make(chan error, count)
chPort := make(chan int, count)
chData := make(chan string, count)
go collectData(chPort, chData, chError)
WAIT_FOR_DATA:
for {
select {
case port := <-chPort:
// data collection server is ready, start CarbonOutput
config.Address = fmt.Sprintf("127.0.0.1:%d", port)
config.Protocol = protocol
err := carbonOutput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
wg.Add(1)
carbonOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
// Send the pack.
inChan <- pack
case data := <-chData:
close(inChan)
wg.Wait() // wait for close to finish, prevents intermittent test failures
c.Expect(data, gs.Equals, expected_data)
break WAIT_FOR_DATA
case err := <-chError:
// fail
c.Assume(err, gs.IsNil)
return
}
}
})
})
}
doit("tcp", collectDataTCP)
doit("udp", collectDataUDP)
}
func HttpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
var (
reqMethod string
reqBody string
reqHeader http.Header
handleWg sync.WaitGroup
runWg sync.WaitGroup
delay bool
)
handler := new(_test_handler)
handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) {
defer handleWg.Done()
if delay {
time.Sleep(time.Duration(2) * time.Millisecond)
}
// Capture request body for test assertions.
p := make([]byte, req.ContentLength)
_, err := req.Body.Read(p)
c.Expect(err.Error(), gs.Equals, "EOF")
reqBody = string(p)
// Capture request method and headers for test assertions.
reqMethod = req.Method
reqHeader = req.Header
// Respond with either a response body or a response code.
if len(handler.respBody) > 0 {
rw.Write([]byte(handler.respBody))
} else {
rw.WriteHeader(handler.respCode)
}
flusher := rw.(http.Flusher)
flusher.Flush()
}
oth := ts.NewOutputTestHelper(ctrl)
encoder := new(plugins.PayloadEncoder)
encConfig := new(plugins.PayloadEncoderConfig)
err := encoder.Init(encConfig)
c.Expect(err, gs.IsNil)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
c.Specify("An HttpOutput", func() {
httpOutput := new(HttpOutput)
config := httpOutput.ConfigStruct().(*HttpOutputConfig)
c.Specify("barfs on bogus URLs", func() {
config.Address = "one-two-three-four"
err := httpOutput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("that is started", func() {
server := httptest.NewServer(handler)
defer server.Close()
runOutput := func() {
httpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
runWg.Done()
}
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
payload := "this is the payload"
pack.Message.SetPayload(payload)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(
[]byte(payload), nil)
config.Address = server.URL
handler.respBody = "Response Body"
c.Specify("makes http POST requests by default", func() {
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(reqMethod, gs.Equals, "POST")
})
c.Specify("makes http PUT requests", func() {
config.Method = "put"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(reqMethod, gs.Equals, "PUT")
})
c.Specify("makes http GET requests", func() {
config.Method = "get"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, "")
c.Expect(reqMethod, gs.Equals, "GET")
})
c.Specify("correctly passes headers along", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(reqBody, gs.Equals, payload)
c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"]))
c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"]))
})
c.Specify("uses http auth when specified", func() {
config.Username = "******"
config.Password = "******"
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
auth := reqHeader.Get("Authorization")
c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue)
decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:])
c.Expect(err, gs.IsNil)
c.Expect(string(decodedAuth), gs.Equals, "user:pass")
})
c.Specify("logs error responses", func() {
handler.respBody = ""
handler.respCode = 500
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
var errMsg string
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
func(err error) {
errMsg = err.Error()
})
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(strings.HasPrefix(errMsg,
"HTTP Error code returned: 500"), gs.IsTrue)
})
c.Specify("honors http timeout interval", func() {
config.HttpTimeout = 1 // 1 millisecond
err := httpOutput.Init(config)
c.Expect(err, gs.IsNil)
var errMsg string
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
func(err error) {
errMsg = err.Error()
})
delay = true
runWg.Add(1)
go runOutput()
handleWg.Add(1)
inChan <- pack
close(inChan)
handleWg.Wait()
runWg.Wait()
c.Expect(strings.Contains(errMsg, "use of closed network connection"),
gs.IsTrue)
})
})
})
}
func CarbonOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := NewCarbonTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
// make test data
const count = 5
lines := make([]string, count)
baseTime := time.Now().UTC().Add(-10 * time.Second)
for i := 0; i < count; i++ {
statName := fmt.Sprintf("stats.name.%d", i)
statTime := baseTime.Add(time.Duration(i) * time.Second)
lines[i] = fmt.Sprintf("%s %d %d", statName, i*2, statTime.Unix())
}
submit_data := strings.Join(lines, "\n")
expected_data := submit_data + "\n"
// a helper to make new test packs
newpack := func() *PipelinePack {
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Message.SetPayload(submit_data)
pack.QueueCursor = "queuecursor"
return pack
}
c.Specify("A CarbonOutput ", func() {
inChan := make(chan *PipelinePack, 1)
output := new(CarbonOutput)
config := output.ConfigStruct().(*CarbonOutputConfig)
pack := newpack()
errChan := make(chan error, 1)
connChan := make(chan net.Conn, 1)
dataChan := make(chan string, count)
startOutput := func(output *CarbonOutput, oth *CarbonTestHelper) {
errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
}
// collectData waits for data to come in on the provided connection.
// It "returns" data using channels, either with an error on the
// errChan or data on the dataChan.
collectData := func(conn net.Conn) {
b := make([]byte, 10000)
n, err := conn.Read(b)
if err != nil {
errChan <- err
return
}
dataChan <- string(b[0:n])
}
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
var (
conn net.Conn
err error
)
oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)
c.Specify("using TCP", func() {
var listener net.Listener
config.Protocol = "tcp"
listenerChan := make(chan net.Listener, 1)
// startListener starts a TCP server waiting for data. It
// "returns" data over channels. First it will *either* return an
// error on the errChan *or* the listener on the listenerChan. If
// listening succeeds it will wait for a connection attempt,
// either returning an error on the errChan or the connection on
// the connChan.
startListener := func() {
tcpaddr, err := net.ResolveTCPAddr("tcp", ":0")
if err != nil {
errChan <- err
return
}
listener, err := net.ListenTCP("tcp", tcpaddr)
if err != nil {
errChan <- err
return
}
listenerChan <- listener
conn, err := listener.Accept()
if err != nil {
errChan <- err
return
}
connChan <- conn
}
go startListener()
select {
case err = <-errChan:
// If we get an error herer it won't be nil, but we use
// c.Assume to trigger the test failure.
c.Assume(err, gs.IsNil)
case listener = <-listenerChan:
defer listener.Close()
}
c.Specify("writes to the network", func() {
config.Address = fmt.Sprintf("127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port)
err = output.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
go startOutput(output, oth)
select {
case err = <-errChan:
// If we get an error herer it won't be nil, but we use
// c.Assume to trigger the test failure.
c.Assume(err, gs.IsNil)
case conn = <-connChan:
defer conn.Close()
}
go collectData(conn)
select {
case err = <-errChan:
// If we get an error herer it won't be nil, but we use
// c.Assume to trigger the test failure.
c.Assume(err, gs.IsNil)
case data := <-dataChan:
c.Expect(data, gs.Equals, expected_data)
}
close(inChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
c.Specify("using UDP", func() {
config.Protocol = "udp"
// startListener starts a UDP server waiting for data. It
// "returns" data over channels. It will either return a net.Conn
// on the connChan or an error on the errChan.
startListener := func() {
udpAddr, err := net.ResolveUDPAddr("udp", ":0")
if err != nil {
errChan <- err
return
}
conn, err := net.ListenUDP("udp", udpAddr)
if err != nil {
errChan <- err
return
}
connChan <- conn
}
go startListener()
select {
case err = <-errChan:
c.Assume(err, gs.IsNil)
case conn = <-connChan:
defer conn.Close()
}
c.Specify("writes to the network", func() {
config.Address = fmt.Sprintf("127.0.0.1:%d", conn.LocalAddr().(*net.UDPAddr).Port)
err = output.Init(config)
c.Assume(err, gs.IsNil)
inChan <- pack
go startOutput(output, oth)
go collectData(conn)
select {
case err = <-errChan:
// If we get an error herer it won't be nil, but we use
// c.Assume to trigger the test failure.
c.Assume(err, gs.IsNil)
case data := <-dataChan:
c.Expect(data, gs.Equals, expected_data)
}
close(inChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("splits packets that are too long", func() {
config.Address = fmt.Sprintf("127.0.0.1:%d", conn.LocalAddr().(*net.UDPAddr).Port)
err = output.Init(config)
c.Assume(err, gs.IsNil)
output.bufSplitSize = 1 // Forces a separate packet for each stat.
inChan <- pack
go startOutput(output, oth)
for i := 0; i < count; i++ {
go collectData(conn)
select {
case err = <-errChan:
// If we get an error herer it won't be nil, but we use
// c.Assume to trigger the test failure.
c.Assume(err, gs.IsNil)
case data := <-dataChan:
c.Expect(data, gs.Equals, lines[i]+"\n")
}
}
close(inChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
}
func SmtpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
c.Specify("A SmtpOutput", func() {
smtpOutput := new(SmtpOutput)
config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
config.SendTo = []string{"root"}
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
runnerName.Return("SmtpOutput")
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
encCall := oth.MockOutputRunner.EXPECT().Encode(pack)
c.Specify("send email payload message", func() {
err := smtpOutput.Init(config)
c.Assume(err, gs.IsNil)
smtpOutput.sendFunction = testSendMail
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
encCall.Return(encoder.Encode(pack))
wg.Add(1)
go func() {
smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait()
})
})
c.Specify("SmtpOutput Message Body Encoding", func() {
smtpOutput := new(SmtpOutput)
chars := "123456789012345678901234567890123456789012345678901234567"
charsE := "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3"
examples := [][]string{
{"Hello", "SGVsbG8="},
{chars, charsE},
{chars + chars, charsE + "\r\n" + charsE},
{chars + chars + "Hello", charsE + "\r\n" + charsE + "\r\n" + "SGVsbG8="},
{"", ""},
{"1", "MQ=="},
}
for _, example := range examples {
smtpOutput.encodeFullMsg([]byte(example[0]))
c.Expect(string(smtpOutput.fullMsg), gs.Equals, example[1])
}
})
// // Use this test with a real server
// c.Specify("Real SmtpOutput output", func() {
// smtpOutput := new(SmtpOutput)
// config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
// config.SendTo = []string{"root"}
// msg := pipeline_ts.GetTestMessage()
// pack := NewPipelinePack(pConfig.InputRecycleChan())
// pack.Message = msg
// pack.Decoded = true
// inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
// inChanCall.Return(inChan)
// runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
// runnerName.Return("SmtpOutput")
// oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
// encCall := oth.MockOutputRunner.EXPECT().Encode(pack)
// c.Specify("send a real email essage", func() {
// err := smtpOutput.Init(config)
// c.Assume(err, gs.IsNil)
// outStr := "Write me out to the network"
// pack.Message.SetPayload(outStr)
// encCall.Return(encoder.Encode(pack))
// go func() {
// wg.Add(1)
// smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
// wg.Done()
// }()
// inChan <- pack
// time.Sleep(1000) // allow time for the message output
// close(inChan)
// wg.Wait()
// // manually check the mail
// })
// })
}
func LogstreamerInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
here, _ := os.Getwd()
dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
c.Expect(tmpErr, gs.Equals, nil)
defer func() {
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
config := NewPipelineConfig(globals)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A LogstreamerInput", func() {
lsInput := &LogstreamerInput{pConfig: config}
lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
lsiConfig.LogDirectory = dirPath
lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
lsiConfig.Differentiator = []string{"logfile"}
lsiConfig.Priority = []string{"^Seq"}
lsiConfig.Decoder = "decoder-name"
c.Specify("w/ no translation map", func() {
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
c.Expect(len(lsInput.plugins), gs.Equals, 1)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 5 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
"-"+lsiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
runOutChan := make(chan error, 1)
go func() {
err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
runOutChan <- err
}()
d, _ := time.ParseDuration("5s")
timeout := time.After(d)
timed := false
for x := 0; x < numLines; x++ {
select {
case <-ith.DecodeChan:
case <-timeout:
timed = true
x += numLines
}
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lsInput.Stop()
c.Expect(timed, gs.Equals, false)
c.Expect(<-runOutChan, gs.Equals, nil)
})
})
c.Specify("with a translation map", func() {
lsiConfig.Translation = make(ls.SubmatchTranslationMap)
lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)
c.Specify("allows len 1 translation map for 'missing'", func() {
lsiConfig.Translation["Seq"]["missing"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
})
c.Specify("doesn't allow len 1 map for other keys", func() {
lsiConfig.Translation["Seq"]["missin"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"A translation map with one entry ('Seq') must be specifying a "+
"'missing' key.")
})
})
})
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
tmpFileName)
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
toString := func(outData interface{}) string {
return string(*(outData.(*[]byte)))
}
c.Specify("correctly formats text output", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 20)
c.Specify("by default", func() {
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
})
c.Specify("w/ a prepended timestamp when specified", func() {
fileOutput.prefix_ts = true
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
payload := *msg.Payload
c.Expect(strContents, pipeline_ts.StringContains, payload)
c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
})
c.Specify("even when payload is nil", func() {
pack.Message.Payload = nil
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
strContents := toString(&outData)
c.Expect(strContents, gs.Equals, "\n")
})
c.Specify("payload is nil and with a timestamp", func() {
pack.Message.Payload = nil
fileOutput.prefix_ts = true
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
c.Expect(strings.HasPrefix(strContents, todayStr), gs.IsTrue)
c.Expect(strings.HasSuffix(strContents, " \n"), gs.IsTrue)
})
})
c.Specify("correctly formats JSON output", func() {
config.Format = "json"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified", func() {
fileOutput.handleMessage(pack, &outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
})
c.Specify("and with a timestamp", func() {
fileOutput.prefix_ts = true
fileOutput.handleMessage(pack, &outData)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(strContents, pipeline_ts.StringContains, string(msgJson)+"\n")
c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
})
})
c.Specify("correctly formats protocol buffer stream output", func() {
config.Format = "protobufstream"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified and timestamp ignored", func() {
fileOutput.prefix_ts = true
err := fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = tmpdir
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
config.Path = filepath.Join(tmpFilePath, "subfile")
err := fileOutput.Init(config)
defer os.Remove(config.Path)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(tmpFilePath)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
runtime.Gosched() // Yield so we can overwrite the timerChan.
fileOutput.timerChan = timerChan
}
cleanUp := func() {
close(inChan)
wg.Done()
}
c.Specify("honors flush interval", func() {
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case _ = <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}
func UdpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
config := &UdpInputConfig{}
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
hbytes, _ := proto.Marshal(header)
buf := encodeMessage(hbytes, mbytes)
bytesChan := make(chan []byte, 1)
ith.MockInputRunner.EXPECT().Name().Return("mock_name")
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().GetRemainingData().AnyTimes()
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
nil).AnyTimes()
splitCall.Do(func(conn net.Conn, del Deliverer) {
recd := make([]byte, 65536)
n, _ := conn.Read(recd)
recd = recd[:n]
bytesChan <- recd
})
c.Specify("using a udp address", func() {
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
config.Net = "udp"
config.Address = ith.AddrStr
err := udpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
c.Specify("passes the connection to SplitStream", func() {
go udpInput.Run(ith.MockInputRunner, ith.MockHelper)
conn, err := net.Dial("udp", ith.AddrStr)
c.Assume(err, gs.IsNil)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
conn.Close()
recd := <-bytesChan
c.Expect(string(recd), gs.Equals, string(buf))
udpInput.Stop()
})
})
if runtime.GOOS != "windows" {
c.Specify("using a unix datagram socket", func() {
tmpDir, err := ioutil.TempDir("", "heka-socket")
c.Assume(err, gs.IsNil)
unixPath := filepath.Join(tmpDir, "unixgram-socket")
ith.AddrStr = unixPath
config.Net = "unixgram"
config.Address = ith.AddrStr
err = udpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UnixConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, unixPath)
c.Specify("passes the socket to SplitStream", func() {
go udpInput.Run(ith.MockInputRunner, ith.MockHelper)
unixAddr, err := net.ResolveUnixAddr("unixgram", unixPath)
c.Assume(err, gs.IsNil)
conn, err := net.DialUnix("unixgram", nil, unixAddr)
c.Assume(err, gs.IsNil)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
conn.Close()
recd := <-bytesChan
c.Expect(string(recd), gs.Equals, string(buf))
udpInput.Stop()
})
})
}
})
}
func OutputRunnerSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockHelper := NewMockPluginHelper(ctrl)
c.Specify("A runner", func() {
stopoutputTimes = 0
pc := NewPipelineConfig(nil)
pluginGlobals := new(PluginGlobals)
c.Specify("restarts a plugin on the first time only", func() {
pluginGlobals.Retries = RetryOptions{
MaxDelay: "1us",
Delay: "1us",
MaxJitter: "1us",
MaxRetries: 1,
}
pw := NewPluginWrapper("stoppingOutput", pc)
pw.ConfigCreator = func() interface{} { return nil }
pw.PluginCreator = func() interface{} { return new(StoppingOutput) }
output := new(StoppingOutput)
pc.outputWrappers = make(map[string]*PluginWrapper)
pc.outputWrappers["stoppingOutput"] = pw
oRunner := NewFORunner("stoppingOutput", output, pluginGlobals, 10)
var wg sync.WaitGroup
cfgCall := mockHelper.EXPECT().PipelineConfig()
cfgCall.Return(pc)
wg.Add(1)
oRunner.Start(mockHelper, &wg) // no panic => success
wg.Wait()
c.Expect(stopoutputTimes, gs.Equals, 2)
})
c.Specify("restarts plugin and resumes feeding it", func() {
pluginGlobals.Retries = RetryOptions{
MaxDelay: "1us",
Delay: "1us",
MaxJitter: "1us",
MaxRetries: 4,
}
pw := NewPluginWrapper("stoppingresumeOutput", pc)
pw.ConfigCreator = func() interface{} { return nil }
pw.PluginCreator = func() interface{} { return new(StopResumeOutput) }
output := new(StopResumeOutput)
pc.outputWrappers = make(map[string]*PluginWrapper)
pc.outputWrappers["stoppingresumeOutput"] = pw
oRunner := NewFORunner("stoppingresumeOutput", output, pluginGlobals, 10)
var wg sync.WaitGroup
cfgCall := mockHelper.EXPECT().PipelineConfig()
cfgCall.Return(pc)
wg.Add(1)
oRunner.Start(mockHelper, &wg) // no panic => success
wg.Wait()
c.Expect(stopresumerunTimes, gs.Equals, 3)
c.Expect(len(stopresumeHolder), gs.Equals, 2)
c.Expect(stopresumeHolder[1], gs.Equals, "woot")
c.Expect(oRunner.retainPack, gs.IsNil)
})
c.Specify("can exit without causing shutdown", func() {
pluginGlobals.Retries = RetryOptions{MaxRetries: 0}
pw := NewPluginWrapper("stoppingOutput", pc)
pw.ConfigCreator = func() interface{} { return nil }
pw.PluginCreator = func() interface{} { return new(StoppingOutput) }
output := new(StoppingOutput)
pc.outputWrappers = make(map[string]*PluginWrapper)
pc.outputWrappers["stoppingOutput"] = pw
oRunner := NewFORunner("stoppingOutput", output, pluginGlobals, 10)
oRunner.canExit = true
// This pack is for the sending of the terminated message
pack := NewPipelinePack(pc.injectRecycleChan)
pc.injectRecycleChan <- pack
// Feed in a pack to the input so we can verify its been recycled
// after stopping (no leaks)
pack = NewPipelinePack(pc.inputRecycleChan)
oRunner.inChan <- pack
// This is code to emulate the router removing the Output, and
// closing up the outputs inChan channel
go func() {
<-pc.Router().RemoveOutputMatcher()
// We don't close the matcher inChan because its not
// instantiated in the tests
close(oRunner.inChan)
}()
var wg sync.WaitGroup
cfgCall := mockHelper.EXPECT().PipelineConfig()
cfgCall.Return(pc)
wg.Add(1)
oRunner.Start(mockHelper, &wg)
wg.Wait()
c.Expect(stopoutputTimes, gs.Equals, 1)
p := <-pc.router.inChan
c.Expect(p.Message.GetType(), gs.Equals, "heka.terminated")
// This should be 1 because the inChan should be flushed
// and packs should not be leaked
c.Expect(len(pc.inputRecycleChan), gs.Equals, 1)
})
c.Specify("encodes a message", func() {
output := new(StoppingOutput)
or := NewFORunner("test", output, pluginGlobals, 10)
or.encoder = new(_payloadEncoder)
_pack.Message = ts.GetTestMessage()
payload := "Test Payload"
c.Specify("without framing", func() {
result, err := or.Encode(_pack)
c.Expect(err, gs.IsNil)
c.Expect(string(result), gs.Equals, payload)
})
c.Specify("with framing", func() {
or.SetUseFraming(true)
result, err := or.Encode(_pack)
c.Expect(err, gs.IsNil)
i := bytes.IndexByte(result, message.UNIT_SEPARATOR)
c.Expect(i > 3, gs.IsTrue, -1)
c.Expect(string(result[i+1:]), gs.Equals, payload)
header := new(message.Header)
ok := DecodeHeader(result[2:i+1], header)
c.Expect(ok, gs.IsTrue)
c.Expect(header.GetMessageLength(), gs.Equals, uint32(len(payload)))
})
})
})
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
encoder := new(plugins.PayloadEncoder)
encoder.Init(encoder.ConfigStruct())
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("w/ ProtobufEncoder", func() {
encoder := new(ProtobufEncoder)
encoder.Init(nil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
c.Specify("uses framing", func() {
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
})
c.Specify("but not if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go func() {
err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
fileOutput.file.Close()
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
if runtime.GOOS != "windows" {
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = filepath.Join(tmpdir, "out.txt")
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
os.RemoveAll(tmpdir)
})
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
subdir := filepath.Join(os.TempDir(), "subdir")
config.Path = filepath.Join(subdir, "out.txt")
err := fileOutput.Init(config)
defer os.RemoveAll(subdir)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(subdir)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
}
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
runtime.Gosched() // Yield so we can overwrite the timerChan.
fileOutput.timerChan = timerChan
}
cleanUp := func() {
close(inChan)
fileOutput.file.Close()
wg.Done()
}
c.Specify("honors flush interval", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case _ = <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
out, err := encoder.Encode(pack2)
oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}
func TcpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
c.Specify("A TcpInput", func() {
tcpInput := &TcpInput{}
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
}
bytesChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var srDoneWG sync.WaitGroup
startServer := func() {
srDoneWG.Add(1)
ith.MockInputRunner.EXPECT().Name().Return("mock_name")
ith.MockInputRunner.EXPECT().NewDeliverer(gomock.Any()).Return(ith.MockDeliverer)
ith.MockDeliverer.EXPECT().Done()
ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
ith.MockSplitterRunner.EXPECT().Done().Do(func() {
srDoneWG.Done()
})
// splitCall gets called twice. The first time it returns nil, the
// second time it returns io.EOF.
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
ith.MockDeliverer).AnyTimes()
splitCall.Do(func(conn net.Conn, del Deliverer) {
recd, _ := ioutil.ReadAll(conn)
bytesChan <- recd
splitCall.Return(io.EOF)
})
err := tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}
c.Specify("not using TLS", func() {
err := tcpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(tcpInput.listener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
c.Specify("accepts connections and passes them to the splitter", func() {
go startServer()
data := []byte("THIS IS THE DATA")
outConn, err := net.Dial("tcp", ith.AddrStr)
c.Assume(err, gs.IsNil)
_, err = outConn.Write(data)
c.Expect(err, gs.IsNil)
outConn.Close()
recd := <-bytesChan
c.Expect(err, gs.IsNil)
c.Expect(string(recd), gs.Equals, string(data))
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
})
c.Specify("using TLS", func() {
config.UseTls = true
c.Specify("fails to init w/ missing key or cert file", func() {
config.Tls = TlsConfig{}
err := tcpInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"TLS config requires both cert_file and key_file value.")
})
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
}
c.Specify("accepts connections and passes them to the splitter", func() {
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go startServer()
data := []byte("This is a test.")
clientConfig := new(tls.Config)
clientConfig.InsecureSkipVerify = true
outConn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Assume(err, gs.IsNil)
outConn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
n, err := outConn.Write(data)
c.Expect(err, gs.IsNil)
c.Expect(n, gs.Equals, len(data))
outConn.Close()
recd := <-bytesChan
c.Expect(err, gs.IsNil)
c.Expect(string(recd), gs.Equals, string(data))
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
c.Specify("doesn't accept connections below specified min TLS version", func() {
config.Tls.MinVersion = "TLS12"
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go startServer()
clientConfig := &tls.Config{
InsecureSkipVerify: true,
MaxVersion: tls.VersionTLS11,
}
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(conn, gs.IsNil)
c.Expect(err, gs.Not(gs.IsNil))
<-bytesChan
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
})
})
}
func ProcessDirectoryInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
err := pConfig.RegisterDefault("NullSplitter")
c.Assume(err, gs.IsNil)
c.Specify("A ProcessDirectoryInput", func() {
pdiInput := ProcessDirectoryInput{}
pdiInput.SetPipelineConfig(pConfig)
config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig)
workingDir, err := os.Getwd()
c.Assume(err, gs.IsNil)
config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes")
// `Ticker` is the last thing called during the setup part of the
// input's `Run` method, so it triggers a waitgroup that tests can
// wait on when they need to ensure initialization has finished.
var started sync.WaitGroup
started.Add(1)
tickChan := make(chan time.Time, 1)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do(
func() {
started.Done()
})
// Similarly we use a waitgroup to signal when LogMessage has been
// called to know when reloads have completed. Warning: If you call
// expectLogMessage with a msg that is never passed to LogMessage and
// then you call loaded.Wait() then your test will hang and never
// complete.
var loaded sync.WaitGroup
expectLogMessage := func(msg string) {
loaded.Add(1)
ith.MockInputRunner.EXPECT().LogMessage(msg).Do(
func(msg string) {
loaded.Done()
})
}
// Same name => same content.
paths := []string{
filepath.Join(config.ProcessDir, "100", "h0.toml"),
filepath.Join(config.ProcessDir, "100", "h1.toml"),
filepath.Join(config.ProcessDir, "200", "h0.toml"),
filepath.Join(config.ProcessDir, "300", "h1.toml"),
}
copyFile := func(src, dest string) {
inFile, err := os.Open(src)
c.Assume(err, gs.IsNil)
outFile, err := os.Create(dest)
c.Assume(err, gs.IsNil)
_, err = io.Copy(outFile, inFile)
c.Assume(err, gs.IsNil)
inFile.Close()
outFile.Close()
}
err = pdiInput.Init(config)
c.Expect(err, gs.IsNil)
for _, p := range paths {
expectLogMessage("Added: " + p)
}
go pdiInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
pdiInput.Stop()
for _, entry := range pdiInput.inputs {
entry.ir.Input().Stop()
}
}()
started.Wait()
c.Specify("loads scheduled jobs", func() {
pathIndex := func(name string) (i int) {
var p string
for i, p = range paths {
if name == p {
return
}
}
return -1
}
for name, entry := range pdiInput.inputs {
i := pathIndex(name)
// Make sure each file path got registered.
c.Expect(i, gs.Not(gs.Equals), -1)
dirName := filepath.Base(filepath.Dir(name))
dirInt, err := strconv.Atoi(dirName)
c.Expect(err, gs.IsNil)
// And that the ticker interval was read correctly.
c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval)
}
})
c.Specify("discovers and adds a new job", func() {
// Copy one of the files to register a new process.
newPath := filepath.Join(config.ProcessDir, "300", "h0.toml")
copyFile(paths[0], newPath)
defer func() {
err := os.Remove(newPath)
c.Assume(err, gs.IsNil)
}()
// Set up expectations and trigger process dir reload.
expectLogMessage("Added: " + newPath)
tickChan <- time.Now()
loaded.Wait()
// Make sure our plugin was loaded.
c.Expect(len(pdiInput.inputs), gs.Equals, 5)
newEntry, ok := pdiInput.inputs[newPath]
c.Expect(ok, gs.IsTrue)
c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300))
})
c.Specify("removes a deleted job", func() {
err := os.Remove(paths[3])
c.Assume(err, gs.IsNil)
defer func() {
copyFile(paths[1], paths[3])
}()
// Set up expectations and trigger process dir reload.
expectLogMessage("Removed: " + paths[3])
tickChan <- time.Now()
loaded.Wait()
// Make sure our plugin was deleted.
c.Expect(len(pdiInput.inputs), gs.Equals, 3)
})
c.Specify("notices a changed job", func() {
// Overwrite one job w/ a slightly different one.
copyFile(paths[0], paths[3])
defer copyFile(paths[1], paths[3])
// Set up expectations and trigger process dir reload.
expectLogMessage("Removed: " + paths[3])
expectLogMessage("Added: " + paths[3])
tickChan <- time.Now()
loaded.Wait()
// Make sure the new config was loaded.
c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals,
"hello world\n")
})
})
}
func HekaFramingSpec(c gs.Context) {
c.Specify("A HekaFramingSplitter", func() {
splitter := &HekaFramingSplitter{}
config := splitter.ConfigStruct().(*HekaFramingSplitterConfig)
sRunner := makeSplitterRunner("HekaFramingSplitter", splitter)
c.Specify("splits records", func() {
b := []byte("\x1e\x02\x08\x3e\x1f\x0a\x10\x90\x1d\x56\x27\xec\x49\x4c\x8f\xba\x8e\x84\x9b\xaa\xf7\xa6\xf6\x10\xa6\x97\x8a\x8f\xb6\xc1\xae\x8e\x13\x1a\x09\x68\x65\x6b\x61\x62\x65\x6e\x63\x68\x28\x06\x3a\x03\x30\x2e\x38\x40\xbf\xe5\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x1e\x02\x08\x3e\x1f\x0a\x10\x90\x1d\x56\x27\xec\x49\x4c\x8f\xba\x8e\x84\x9b\xaa\xf7\xa6\xf6\x10\xa6\x97\x8a\x8f\xb6\xc1\xae\x8e\x13\x1a\x09\x68\x65\x6b\x61\x62\x65\x6e\x63\x68\x28\x06\x3a\x03\x30\x2e\x38\x40\xbf\xe5\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30BOGUS\x1e\x02\x08\x3e\x1f\x0a\x10\x90\x1d\x56\x27\xec\x49\x4c\x8f\xba\x8e\x84\x9b\xaa\xf7\xa6\xf6\x10\xa6\x97\x8a\x8f\xb6\xc1\xae\x8e\x13\x1a\x09\x68\x65\x6b\x61\x62\x65\x6e\x63\x68\x28\x06\x3a\x03\x30\x2e\x38\x40\xbf\xe5\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30BOGUS\x1e\x02\x08")
reader := bytes.NewReader(b)
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
n, record, err := sRunner.GetRecordFromStream(reader)
c.Expect(n, gs.Equals, 67)
c.Expect(err, gs.IsNil)
c.Expect(string(record), gs.Equals, string(b[:67]))
n, record, err = sRunner.GetRecordFromStream(reader)
c.Expect(n, gs.Equals, 67)
c.Expect(err, gs.IsNil)
c.Expect(string(record), gs.Equals, string(b[67:134]))
n, record, err = sRunner.GetRecordFromStream(reader)
c.Expect(n, gs.Equals, 72) // skips the invalid 'BOGUS' data
c.Expect(err, gs.IsNil)
c.Expect(string(record), gs.Equals, string(b[139:206]))
n, record, err = sRunner.GetRecordFromStream(reader) // trigger the need to read more data
c.Expect(n, gs.Equals, 5)
c.Expect(err, gs.IsNil)
c.Expect(len(record), gs.Equals, 0)
n, record, err = sRunner.GetRecordFromStream(reader) // hit the EOF
c.Expect(n, gs.Equals, 0)
c.Expect(err, gs.Equals, io.EOF)
c.Expect(len(record), gs.Equals, 0)
})
c.Specify("correctly handles invalid header (no unit separator)", func() {
b := []byte("\x1e\x02\x08\x3e\xff\x1e\x02\x08\x3e\x1f\x0a\x10\x90\x1d\x56\x27\xec\x49\x4c\x8f\xba\x8e\x84\x9b\xaa\xf7\xa6\xf6\x10\xa6\x97\x8a\x8f\xb6\xc1\xae\x8e\x13\x1a\x09\x68\x65\x6b\x61\x62\x65\x6e\x63\x68\x28\x06\x3a\x03\x30\x2e\x38\x40\xbf\xe5\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30")
reader := bytes.NewReader(b)
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
n, record, err := sRunner.GetRecordFromStream(reader)
c.Expect(n, gs.Equals, 72)
c.Expect(err, gs.IsNil)
c.Expect(string(record), gs.Equals, string(b[5:]))
})
c.Specify("using authentication", func() {
key := "testkey"
config.Signers = map[string]Signer{"test_1": {key}}
signer := "test"
recycleChan := make(chan *PipelinePack, 1)
pack := NewPipelinePack(recycleChan)
msg := ts.GetTestMessage()
mbytes, _ := proto.Marshal(msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
c.Specify("authenticates MD5 signed message", func() {
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
framed := encodeMessage(hbytes, mbytes)
unframed := splitter.UnframeRecord(framed, pack)
c.Expect(pack.Signer, gs.Equals, "test")
c.Expect(string(unframed), gs.Equals, string(mbytes))
})
c.Specify("authenticates SHA1 signed message", func() {
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
header.SetHmacHashFunction(message.Header_SHA1)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(sha1.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
framed := encodeMessage(hbytes, mbytes)
unframed := splitter.UnframeRecord(framed, pack)
c.Expect(pack.Signer, gs.Equals, "test")
c.Expect(string(unframed), gs.Equals, string(mbytes))
})
c.Specify("doesn't auth signed message with expired key", func() {
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(11)) // non-existent key version
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
framed := encodeMessage(hbytes, mbytes)
unframed := splitter.UnframeRecord(framed, pack)
c.Expect(pack.Signer, gs.Equals, "")
// The function returns nil, and `unframed == nil` evaluates
// to true, but `gs.IsNil` doesn't work here.
c.Expect(string(unframed), gs.Equals, "")
})
c.Specify("doesn't auth signed message with incorrect hmac", func() {
err := splitter.Init(config)
c.Assume(err, gs.IsNil)
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write([]byte("some bytes"))
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
framed := encodeMessage(hbytes, mbytes)
unframed := splitter.UnframeRecord(framed, pack)
c.Expect(pack.Signer, gs.Equals, "")
// The function returns nil, and `unframed == nil` evaluates
// to true, but `gs.IsNil` doesn't work here.
c.Expect(string(unframed), gs.Equals, "")
})
})
})
}
func ReportSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
chanSize := pConfig.Globals.PluginChanSize
checkForFields := func(c gs.Context, msg *message.Message) {
f0Val, ok := msg.GetFieldValue(f0.GetName())
c.Expect(ok, gs.IsTrue)
c.Expect(f0Val.(int64), gs.Equals, f0.GetValue().(int64))
f1Val, ok := msg.GetFieldValue(f1.GetName())
c.Expect(ok, gs.IsTrue)
c.Expect(f1Val.(string), gs.Equals, f1.GetValue().(string))
}
hasChannelData := func(msg *message.Message) (ok bool) {
capVal, _ := msg.GetFieldValue("InChanCapacity")
lenVal, _ := msg.GetFieldValue("InChanLength")
var i int64
if i, ok = capVal.(int64); !ok {
return
}
if ok = (i == int64(chanSize)); !ok {
return
}
if i, ok = lenVal.(int64); !ok {
return
}
ok = (i == int64(0))
return
}
fName := "counter"
filter := new(CounterFilter)
fRunner := NewFORunner(fName, filter, nil, chanSize)
var err error
fRunner.matcher, err = NewMatchRunner("Type == ''", "", fRunner, chanSize)
c.Assume(err, gs.IsNil)
fRunner.matcher.inChan = make(chan *PipelinePack, chanSize)
leakCount := 10
fRunner.SetLeakCount(leakCount)
iName := "stat_accum"
input := new(StatAccumInput)
iRunner := NewInputRunner(iName, input, nil, false)
c.Specify("`PopulateReportMsg`", func() {
msg := ts.GetTestMessage()
c.Specify("w/ a filter", func() {
err := PopulateReportMsg(fRunner, msg)
c.Assume(err, gs.IsNil)
c.Specify("invokes `ReportMsg` on the filter", func() {
checkForFields(c, msg)
})
c.Specify("adds the channel data", func() {
c.Expect(hasChannelData(msg), gs.IsTrue)
})
c.Specify("has its leak count set properly", func() {
leakVal, ok := msg.GetFieldValue("LeakCount")
c.Assume(ok, gs.IsTrue)
i, ok := leakVal.(int64)
c.Assume(ok, gs.IsTrue)
c.Expect(int(i), gs.Equals, leakCount)
})
})
c.Specify("w/ an input", func() {
err := PopulateReportMsg(iRunner, msg)
c.Assume(err, gs.IsNil)
c.Specify("invokes `ReportMsg` on the input", func() {
checkForFields(c, msg)
})
c.Specify("doesn't add any channel data", func() {
capVal, ok := msg.GetFieldValue("InChanCapacity")
c.Expect(capVal, gs.IsNil)
c.Expect(ok, gs.IsFalse)
lenVal, ok := msg.GetFieldValue("InChanLength")
c.Expect(lenVal, gs.IsNil)
c.Expect(ok, gs.IsFalse)
})
})
})
c.Specify("PipelineConfig", func() {
pc := NewPipelineConfig(nil)
// Initialize all of the PipelinePacks that we'll need
pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)
pc.FilterRunners = map[string]FilterRunner{fName: fRunner}
pc.InputRunners = map[string]InputRunner{iName: iRunner}
c.Specify("returns full set of accurate reports", func() {
reportChan := make(chan *PipelinePack)
go pc.reports(reportChan)
reports := make(map[string]*PipelinePack)
for r := range reportChan {
iName, ok := r.Message.GetFieldValue("name")
c.Expect(ok, gs.IsTrue)
name, ok := iName.(string)
c.Expect(ok, gs.IsTrue)
c.Expect(name, gs.Not(gs.Equals), "MISSING")
reports[name] = r
pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)
}
fReport := reports[fName]
c.Expect(fReport, gs.Not(gs.IsNil))
checkForFields(c, fReport.Message)
c.Expect(hasChannelData(fReport.Message), gs.IsTrue)
iReport := reports[iName]
c.Expect(iReport, gs.Not(gs.IsNil))
checkForFields(c, iReport.Message)
recycleReport := reports["inputRecycleChan"]
c.Expect(recycleReport, gs.Not(gs.IsNil))
capVal, ok := recycleReport.Message.GetFieldValue("InChanCapacity")
c.Expect(ok, gs.IsTrue)
c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))
injectReport := reports["injectRecycleChan"]
c.Expect(injectReport, gs.Not(gs.IsNil))
capVal, ok = injectReport.Message.GetFieldValue("InChanCapacity")
c.Expect(ok, gs.IsTrue)
c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))
routerReport := reports["Router"]
c.Expect(routerReport, gs.Not(gs.IsNil))
c.Expect(hasChannelData(routerReport.Message), gs.IsTrue)
})
})
}
func TcpOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests")
defer func() {
ctrl.Finish()
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.Equals, nil)
}()
pConfig := NewPipelineConfig(nil)
c.Specify("TcpOutput", func() {
origBaseDir := Globals().BaseDir
Globals().BaseDir = tmpDir
defer func() {
Globals().BaseDir = origBaseDir
}()
tcpOutput := new(TcpOutput)
tcpOutput.SetName("test")
config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
tcpOutput.Init(config)
tickChan := make(chan time.Time)
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan)
encoder := new(ProtobufEncoder)
encoder.Init(nil)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
outStr := "Write me out to the network"
newpack := NewPipelinePack(nil)
newpack.Message = msg
newpack.Decoded = true
newpack.Message.SetPayload(outStr)
matchBytes, err := proto.Marshal(newpack.Message)
c.Expect(err, gs.IsNil)
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
c.Specify("doesn't use framing w/o ProtobufEncoder", func() {
encoder := new(plugins.PayloadEncoder)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
c.Specify("doesn't use framing if config says not to", func() {
useFraming := false
config.UseFraming = &useFraming
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go func() {
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
close(inChan)
wg.Wait()
// We should fail if SetUseFraming is called since we didn't
// EXPECT it.
})
c.Specify("writes out to the network", func() {
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
ch <- "ready"
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch // wait for server
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()
pack.Message.SetPayload(outStr)
wg.Add(1)
go func() {
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
inChan <- pack
result = <-ch
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(1))
c.Expect(result, gs.Equals, string(matchBytes))
close(inChan)
wg.Wait() // wait for output to finish shutting down
})
c.Specify("far end not initially listening", func() {
oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
pack.Message.SetPayload(outStr)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().SetUseFraming(true)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()
wg.Add(1)
go func() {
err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
c.Expect(err, gs.IsNil)
wg.Done()
}()
msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
c.Expect(msgcount, gs.Equals, int64(0))
inChan <- pack
for x := 0; x < 5 && msgcount == 0; x++ {
msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
time.Sleep(time.Duration(100) * time.Millisecond)
}
// After the message is queued start the collector. However, we
// don't have a way guarantee a send attempt has already been made
// and that we are actually exercising the retry code.
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
return
}
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
return
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
conn.Close()
ln.Close()
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch
c.Expect(result, gs.Equals, string(matchBytes))
close(inChan)
wg.Wait() // wait for output to finish shutting down
})
})
}
func UdpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr,
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("UdpInput")
encCall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "UdpInput-ProtobufDecoder")
encCall.Return(ith.Decoder, true)
c.Specify("reads a message from the connection and passes it to the decoder", func() {
hbytes, _ := proto.Marshal(header)
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
c.Assume(err, gs.IsNil)
buf := encodeMessage(hbytes, mbytes)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
udpInput.Stop()
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Decoded, gs.IsFalse)
})
})
c.Specify("A UdpInput Multiline input", func() {
ith.AddrStr = "localhost:55566"
ith.ResolvedAddrStr = "127.0.0.1:55566"
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr,
Decoder: "test",
ParserType: "token"})
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).Times(2)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(2)
ith.MockInputRunner.EXPECT().Name().Return("UdpInput").AnyTimes()
encCall := ith.MockHelper.EXPECT().DecoderRunner("test", "UdpInput-test")
encCall.Return(ith.Decoder, true)
c.Specify("reads two messages from a packet and passes them to the decoder", func() {
go func() {
udpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
c.Assume(err, gs.IsNil)
_, err = conn.Write([]byte("message1\nmessage2\n"))
c.Assume(err, gs.IsNil)
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message1\n")
ith.PackSupply <- ith.Pack
packRef = <-ith.DecodeChan
c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message2\n")
udpInput.Stop()
})
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
mockDRunner := NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
c.Specify("with a valid setup and no decoder", func() {
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a message", func() {
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
})
c.Specify("with a valid setup using a decoder", func() {
decoderName := "defaultDecoder"
config.Decoder = decoderName
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
// Mock up our default decoder runner and decoder.
ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
decCall.Return(mockDRunner, true)
mockDecoder := NewMockDecoder(ctrl)
mockDRunner.EXPECT().Decoder().Return(mockDecoder)
c.Specify("consumes a message", func() {
packs := []*PipelinePack{ith.Pack}
mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)
// Create a channel to send data to the input
// Drop a message on there and close the channel
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the injected packet
ith.MockInputRunner.EXPECT().Inject(gomock.Any())
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
})
c.Specify("consumes a serialized message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Expect the decoded pack
mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)
// Increase the usage since Run decrements it on close
ug.Add(1)
ith.PackSupply <- ith.Pack
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
// Ignore leading 5 bytes of encoded message as thats the header
c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
ith.PackSupply <- ith.Pack
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close
ug.Add(1)
// Expect the close and the InChan calls
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
