func DecodersSpec(c gospec.Context) {
msg := getTestMessage()
c.Specify("A JsonDecoder", func() {
var fmtString = `{"type":"%s","timestamp":%s,"logger":"%s","severity":%d,"payload":"%s","fields":%s,"env_version":"%s","metlog_pid":%d,"metlog_hostname":"%s"}`
timestampJson, err := json.Marshal(msg.Timestamp)
fieldsJson, err := json.Marshal(msg.Fields)
c.Assume(err, gs.IsNil)
jsonString := fmt.Sprintf(fmtString, msg.Type,
timestampJson, msg.Logger, msg.Severity, msg.Payload,
fieldsJson, msg.Env_version, msg.Pid, msg.Hostname)
pipelinePack := getTestPipelinePack()
pipelinePack.MsgBytes = []byte(jsonString)
jsonDecoder := new(JsonDecoder)
c.Specify("can decode a JSON message", func() {
err := jsonDecoder.Decode(pipelinePack)
c.Expect(pipelinePack.Message, gs.Equals, msg)
c.Expect(err, gs.IsNil)
})
c.Specify("returns `fields` as a map", func() {
jsonDecoder.Decode(pipelinePack)
c.Expect(pipelinePack.Message.Fields["foo"], gs.Equals, "bar")
})
c.Specify("returns an error for bogus JSON", func() {
badJson := fmt.Sprint("{{", jsonString)
pipelinePack.MsgBytes = []byte(badJson)
err := jsonDecoder.Decode(pipelinePack)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(pipelinePack.Message.Timestamp.IsZero(), gs.IsTrue)
})
})
c.Specify("A MsgPackDecoder", func() {
msg := getTestMessage()
encoded, err := msgpack.Marshal(msg)
c.Assume(err, gs.IsNil)
decoder := new(MsgPackDecoder)
decoder.Init(nil)
pack := getTestPipelinePack()
c.Specify("decodes a msgpack message", func() {
pack.MsgBytes = encoded
err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(pack.Message, gs.Equals, msg)
})
c.Specify("returns an error for bunk encoding", func() {
bunk := append([]byte{0, 0, 0}, encoded...)
pack.MsgBytes = bunk
err := decoder.Decode(pack)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func ProtobufDecoderSpec(c gospec.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
msg := ts.GetTestMessage()
config := NewPipelineConfig(nil) // Initializes globals.
c.Specify("A ProtobufDecoder", func() {
encoded, err := proto.Marshal(msg)
c.Assume(err, gs.IsNil)
pack := NewPipelinePack(config.inputRecycleChan)
decoder := new(ProtobufDecoder)
decoder.sampleDenominator = 1000 // Since we don't call decoder.Init().
c.Specify("decodes a protobuf message", func() {
pack.MsgBytes = encoded
_, err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(pack.Message, gs.Equals, msg)
v, ok := pack.Message.GetFieldValue("foo")
c.Expect(ok, gs.IsTrue)
c.Expect(v, gs.Equals, "bar")
})
c.Specify("returns an error for bunk encoding", func() {
bunk := append([]byte{0, 0, 0}, encoded...)
pack.MsgBytes = bunk
_, err := decoder.Decode(pack)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func PrimitiveDecodeStrictSpec(c gs.Context) {
var md MetaData
var err error
var tomlBlob = `
ranking = ["Springsteen", "J Geils"]
[bands.Springsteen]
type = "ignore_this"
started = 1973
albums = ["Greetings", "WIESS", "Born to Run", "Darkness"]
not_albums = ["Greetings", "WIESS", "Born to Run", "Darkness"]
[bands.J Geils]
started = 1970
albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"]
`
type band struct {
Started int
Albums []string
}
type classics struct {
Ranking []string
Bands map[string]Primitive
}
// Do the initial decode. Reflection is delayed on Primitive values.
var music classics
md, err = Decode(tomlBlob, &music)
c.Assume(err, gs.IsNil)
// MetaData still includes information on Primitive values.
c.Assume(md.IsDefined("bands", "Springsteen"), gs.IsTrue)
ignore_type := map[string]interface{}{"type": true}
// Decode primitive data into Go values.
for _, artist := range music.Ranking {
// A band is a primitive value, so we need to decode it to get a
// real `band` value.
primValue := music.Bands[artist]
var aBand band
err = PrimitiveDecodeStrict(primValue, &aBand, ignore_type)
if artist == "Springsteen" {
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "Configuration contains key [not_albums] which doesn't exist in struct")
c.Assume(1973, gs.Equals, aBand.Started)
} else {
c.Expect(err, gs.IsNil)
c.Assume(1970, gs.Equals, aBand.Started)
}
}
}
func UdpInputSpecFailure(c gs.Context) {
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{Net: "tcp", Address: "localhost:55565",
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.Not(gs.IsNil))
c.Assume(err.Error(), gs.Equals, "ResolveUDPAddr failed: unknown network tcp\n")
}
func TcpInputSpecFailure(c gs.Context) {
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{Net: "udp", Address: "localhost:55565",
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.Not(gs.IsNil))
c.Assume(err.Error(), gs.Equals, "ListenTCP failed: unknown network udp\n")
}
func UdpInputSpecFailure(c gs.Context) {
udpInput := UdpInput{}
err := udpInput.Init(&UdpInputConfig{
Net: "tcp",
Address: "localhost:55565",
})
c.Assume(err, gs.Not(gs.IsNil))
c.Assume(err.Error(), gs.Equals, "ResolveUDPAddr failed: unknown network tcp\n")
}
func TcpInputSpecFailure(c gs.Context) {
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{
Net: "udp",
Address: "localhost:55565",
})
c.Assume(err, gs.Not(gs.IsNil))
c.Assume(err.Error(), gs.Equals, "ResolveTCPAddress failed: unknown network udp\n")
}
func MessageEqualsSpec(c gospec.Context) {
msg0 := getTestMessage()
msg1Real := *msg0
msg1 := &msg1Real
c.Specify("Messages are equal", func() {
c.Expect(msg0, gs.Equals, msg1)
})
c.Specify("Messages w/ diff int values are not equal", func() {
msg1.Severity--
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ diff string values are not equal", func() {
msg1.Payload = "Something completely different"
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ diff maps are not equal", func() {
msg1.Fields = map[string]interface{}{"sna": "foo"}
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
}
func DecodeSpec(c gs.Context) {
var val simple
md, err := Decode(testSimple, &val)
c.Assume(err, gs.IsNil)
c.Assume(md.IsDefined("Annoying", "Cats", "plato"), gs.IsTrue)
c.Assume(md.IsDefined("Cats", "Stinky"), gs.IsFalse)
var colors = [][]string{[]string{"red", "green", "blue"},
[]string{"cyan", "magenta", "yellow", "black"}}
for ridx, row := range colors {
for cidx, _ := range row {
c.Assume(val.Colors[ridx][cidx], gs.Equals, colors[ridx][cidx])
}
}
c.Assume(val, gs.Not(gs.IsNil))
}
func DashboardOutputSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
c.Specify("A FileOutput", func() {
dashboardOutput := new(DashboardOutput)
config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := path.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.WorkingDirectory = tmpdir
err = dashboardOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
})
}
func DashboardOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
NewPipelineConfig(nil) // Needed for side effect of setting up Globals :P
if runtime.GOOS != "windows" {
c.Specify("A DashboardOutput", func() {
dashboardOutput := new(DashboardOutput)
config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.WorkingDirectory = tmpdir
err = dashboardOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
})
}
}
func JsonPathSpec(c gs.Context) {
c.Specify("JsonPath can read data", func() {
var s = `{
"foo": {
"bar": [
{
"baz": "こんにちわ世界",
"noo": "aaa"
},
{
"maz": "123",
"moo": 256,
"muux": 2.10
}
],
"boo": {
"bag": true,
"bug": false
}
}
}
`
var err error
var json_path *JsonPath
var result interface{}
json_path = new(JsonPath)
err = json_path.SetJsonText(s)
c.Expect(err, gs.IsNil)
result, err = json_path.Find("$.foo.bar[0].baz")
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, "こんにちわ世界")
result, err = json_path.Find("$.foo.bar[0].noo")
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, "aaa")
result, err = json_path.Find("$.foo.bar[1].maz")
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, "123")
result, err = json_path.Find("$.foo.bar[1].moo")
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, "256")
result, err = json_path.Find("$.foo.bar[1].muux")
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, "2.10")
result, err = json_path.Find("$.foo.boo.bag")
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, "true")
result, err = json_path.Find("$.foo.boo.bug")
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, "false")
result, err = json_path.Find("$.foo.bar[99].baz")
c.Expect(err, gs.Not(gs.IsNil))
result, err = json_path.Find("$.badpath")
c.Expect(err, gs.Not(gs.IsNil))
result, err = json_path.Find("badpath")
c.Expect(err, gs.Not(gs.IsNil))
result, err = json_path.Find("$.foo.bar.3428")
c.Expect(err, gs.Not(gs.IsNil))
expected_data := `[{"baz":"こんにちわ世界","noo":"aaa"},{"maz":"123","moo":256,"muux":2.10}]`
result_data, err := json_path.Find("$.foo.bar")
c.Expect(err, gs.IsNil)
c.Expect(result_data, gs.Equals, expected_data)
})
c.Specify("JsonPath doesn't crash on nil data", func() {
var err error
var json_path *JsonPath
json_path = new(JsonPath)
err = json_path.SetJsonText("")
c.Expect(err, gs.Not(gs.IsNil))
// Searches should return an error
result, err := json_path.Find("$.foo.bar.3428")
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "JSON data is nil")
c.Expect(result, gs.Equals, "")
})
c.Specify("JsonPath handles arrays at top level", func() {
var err error
var json_path *JsonPath
json_path = new(JsonPath)
err = json_path.SetJsonText(`["foo"]`)
c.Expect(err, gs.IsNil)
// Searches should return an error
result, err := json_path.Find("$.[0]")
c.Expect(err, gs.IsNil)
c.Expect(result, gs.Equals, "foo")
})
c.Specify("JsonPath handles invalid doubly encoded JSON gracefully", func() {
s := `{
"request":{
"parameters":{
"invites":"[{\"inviteUserId\":\"123\",\"email\":\"[email protected]\",\"phone\":\"123\",\"name\":\"John
Doe\"}]",
"feature":"0"
}
}
}`
var err error
var json_path *JsonPath
json_path = new(JsonPath)
err = json_path.SetJsonText(s)
c.Expect(err, gs.Not(gs.IsNil))
// Searches should return an error
result, err := json_path.Find("$.[0]")
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "JSON data is nil")
c.Expect(result, gs.Equals, "")
})
}
func DashboardOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := pipeline.NewPipelineConfig(nil)
dashboardOutput := new(DashboardOutput)
dashboardOutput.pConfig = pConfig
oth := plugins_ts.NewOutputTestHelper(ctrl)
oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)
errChan := make(chan error, 1)
startOutput := func() {
go func() {
errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
}()
}
if runtime.GOOS != "windows" {
c.Specify("A DashboardOutput", func() {
tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
c.Assume(err, gs.IsNil)
config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
config.WorkingDirectory = tmpdir
c.Specify("Init halts if basedirectory is not writable", func() {
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
err = dashboardOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("that is running", func() {
startedChan := make(chan bool, 1)
defer close(startedChan)
ts := httptest.NewUnstartedServer(nil)
dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
ts.Start()
startedChan <- true
return nil
}
ticker := make(chan time.Time)
inChan := make(chan *pipeline.PipelinePack, 1)
recycleChan := make(chan *pipeline.PipelinePack, 1)
pack := pipeline.NewPipelinePack(recycleChan)
pack.Message = pipeline_ts.GetTestMessage()
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)
err := os.MkdirAll(tmpdir, 0700)
c.Assume(err, gs.IsNil)
defer os.RemoveAll(tmpdir)
dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Noop
})
c.Specify("sets custom http headers", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err = dashboardOutput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = dashboardOutput.server
startOutput()
inChan <- pack
<-startedChan
resp, err := http.Get(ts.URL)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
// Verify headers are there
eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
c.Expect(eq, gs.IsTrue)
eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
c.Expect(eq, gs.IsTrue)
})
close(inChan)
c.Expect(<-errChan, gs.IsNil)
ts.Close()
})
})
}
}
func LogstreamerInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
here, _ := os.Getwd()
dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
c.Expect(tmpErr, gs.Equals, nil)
defer func() {
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
pConfig := NewPipelineConfig(globals)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// Specify localhost, but we're not really going to use the network.
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// Set up mock helper, runner, and pack supply channel.
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
c.Specify("A LogstreamerInput", func() {
lsInput := &LogstreamerInput{pConfig: pConfig}
lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
lsiConfig.LogDirectory = dirPath
lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
lsiConfig.Differentiator = []string{"logfile"}
lsiConfig.Priority = []string{"^Seq"}
c.Specify("w/ no translation map", func() {
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
c.Expect(len(lsInput.plugins), gs.Equals, 1)
// Create pool of packs.
numLines := 5 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs.
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
gomock.Any()).Times(numLines)
line := "boo hoo foo foo"
getRecCall.Return(len(line), []byte(line), nil)
getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
getRecCall.Return(0, make([]byte, 0), io.EOF)
deliverChan := make(chan []byte, 1)
deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
ith.MockDeliverer).Times(numLines)
deliverCall.Do(func(record []byte, del Deliverer) {
deliverChan <- record
})
ith.MockDeliverer.EXPECT().Done()
runOutChan := make(chan error, 1)
go func() {
err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
runOutChan <- err
}()
dur, _ := time.ParseDuration("5s")
timeout := time.After(dur)
timed := false
for x := 0; x < numLines; x++ {
select {
case record := <-deliverChan:
c.Expect(string(record), gs.Equals, line)
case <-timeout:
timed = true
x += numLines
}
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lsInput.Stop()
c.Expect(timed, gs.Equals, false)
c.Expect(<-runOutChan, gs.Equals, nil)
})
})
c.Specify("with a translation map", func() {
lsiConfig.Translation = make(ls.SubmatchTranslationMap)
lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)
c.Specify("allows len 1 translation map for 'missing'", func() {
lsiConfig.Translation["Seq"]["missing"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
})
c.Specify("doesn't allow len 1 map for other keys", func() {
lsiConfig.Translation["Seq"]["missin"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"A translation map with one entry ('Seq') must be specifying a "+
"'missing' key.")
})
})
})
}
func OutputsSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
tmpFileName)
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := getTestMessage()
pack := NewPipelinePack(pConfig.inputRecycleChan)
pack.Message = msg
pack.Decoded = true
toString := func(outData interface{}) string {
return string(*(outData.(*[]byte)))
}
c.Specify("correctly formats text output", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 20)
c.Specify("by default", func() {
fileOutput.handleMessage(pack, &outData)
c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
})
c.Specify("w/ a prepended timestamp when specified", func() {
fileOutput.prefix_ts = true
fileOutput.handleMessage(pack, &outData)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
payload := *msg.Payload
c.Expect(strContents, ts.StringContains, payload)
c.Expect(strContents, ts.StringStartsWith, todayStr)
})
})
c.Specify("correctly formats JSON output", func() {
config.Format = "json"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified", func() {
fileOutput.handleMessage(pack, &outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
})
c.Specify("and with a timestamp", func() {
fileOutput.prefix_ts = true
fileOutput.handleMessage(pack, &outData)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(strContents, ts.StringContains, string(msgJson)+"\n")
c.Expect(strContents, ts.StringStartsWith, todayStr)
})
})
c.Specify("correctly formats protocol buffer stream output", func() {
config.Format = "protobufstream"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified and timestamp ignored", func() {
fileOutput.prefix_ts = true
err := fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = tmpdir
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), ts.StringContains, "-------")
}
})
})
})
c.Specify("A TcpOutput", func() {
tcpOutput := new(TcpOutput)
config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
tcpOutput.connection = ts.NewMockConn(ctrl)
msg := getTestMessage()
pack := NewPipelinePack(pConfig.inputRecycleChan)
pack.Message = msg
pack.Decoded = true
c.Specify("correctly formats protocol buffer stream output", func() {
outBytes := make([]byte, 0, 200)
err := createProtobufStream(pack, &outBytes)
c.Expect(err, gs.IsNil)
b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
c.Expect(bytes.Equal(b, (outBytes)[:len(b)]), gs.IsTrue)
})
c.Specify("writes out to the network", func() {
inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
inChanCall.Return(inChan)
collectData := func(ch chan string) {
ln, err := net.Listen("tcp", "localhost:9125")
if err != nil {
ch <- err.Error()
}
ch <- "ready"
conn, err := ln.Accept()
if err != nil {
ch <- err.Error()
}
b := make([]byte, 1000)
n, _ := conn.Read(b)
ch <- string(b[0:n])
}
ch := make(chan string, 1) // don't block on put
go collectData(ch)
result := <-ch // wait for server
err := tcpOutput.Init(config)
c.Assume(err, gs.IsNil)
outStr := "Write me out to the network"
pack.Message.SetPayload(outStr)
go func() {
wg.Add(1)
tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
wg.Done()
}()
inChan <- pack
close(inChan)
wg.Wait() // wait for close to finish, prevents intermittent test failures
matchBytes := make([]byte, 0, 1000)
err = createProtobufStream(pack, &matchBytes)
c.Expect(err, gs.IsNil)
result = <-ch
c.Expect(result, gs.Equals, string(matchBytes))
})
})
c.Specify("Runner restarts a plugin on the first time only", func() {
pc := new(PipelineConfig)
var pluginGlobals PluginGlobals
pluginGlobals.Retries = RetryOptions{
MaxDelay: "1us",
Delay: "1us",
MaxJitter: "1us",
MaxRetries: 1,
}
pw := &PluginWrapper{
name: "stoppingOutput",
configCreator: func() interface{} { return nil },
pluginCreator: func() interface{} { return new(StoppingOutput) },
}
output := new(StoppingOutput)
pc.outputWrappers = make(map[string]*PluginWrapper)
pc.outputWrappers["stoppingOutput"] = pw
oRunner := NewFORunner("stoppingOutput", output, &pluginGlobals)
var wg sync.WaitGroup
cfgCall := oth.MockHelper.EXPECT().PipelineConfig()
cfgCall.Return(pc)
wg.Add(1)
oRunner.Start(oth.MockHelper, &wg) // no panic => success
wg.Wait()
c.Expect(stopoutputTimes, gs.Equals, 2)
})
c.Specify("Runner restarts plugin and resumes feeding it", func() {
pc := new(PipelineConfig)
var pluginGlobals PluginGlobals
pluginGlobals.Retries = RetryOptions{
MaxDelay: "1us",
Delay: "1us",
MaxJitter: "1us",
MaxRetries: 4,
}
pw := &PluginWrapper{
name: "stoppingresumeOutput",
configCreator: func() interface{} { return nil },
pluginCreator: func() interface{} { return new(StopResumeOutput) },
}
output := new(StopResumeOutput)
pc.outputWrappers = make(map[string]*PluginWrapper)
pc.outputWrappers["stoppingresumeOutput"] = pw
oRunner := NewFORunner("stoppingresumeOutput", output, &pluginGlobals)
var wg sync.WaitGroup
cfgCall := oth.MockHelper.EXPECT().PipelineConfig()
cfgCall.Return(pc)
wg.Add(1)
oRunner.Start(oth.MockHelper, &wg) // no panic => success
wg.Wait()
c.Expect(stopresumerunTimes, gs.Equals, 3)
c.Expect(len(stopresumeHolder), gs.Equals, 2)
c.Expect(stopresumeHolder[1], gs.Equals, "woot")
c.Expect(oRunner.retainPack, gs.IsNil)
})
}
func FileOutputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oth := plugins_ts.NewOutputTestHelper(ctrl)
var wg sync.WaitGroup
inChan := make(chan *PipelinePack, 1)
pConfig := NewPipelineConfig(nil)
c.Specify("A FileOutput", func() {
fileOutput := new(FileOutput)
tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
tmpFileName)
config := fileOutput.ConfigStruct().(*FileOutputConfig)
config.Path = tmpFilePath
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
toString := func(outData interface{}) string {
return string(*(outData.(*[]byte)))
}
c.Specify("correctly formats text output", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 20)
c.Specify("by default", func() {
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
})
c.Specify("w/ a prepended timestamp when specified", func() {
fileOutput.prefix_ts = true
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
payload := *msg.Payload
c.Expect(strContents, pipeline_ts.StringContains, payload)
c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
})
c.Specify("even when payload is nil", func() {
pack.Message.Payload = nil
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
strContents := toString(&outData)
c.Expect(strContents, gs.Equals, "\n")
})
c.Specify("payload is nil and with a timestamp", func() {
pack.Message.Payload = nil
fileOutput.prefix_ts = true
err = fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
c.Expect(strings.HasPrefix(strContents, todayStr), gs.IsTrue)
c.Expect(strings.HasSuffix(strContents, " \n"), gs.IsTrue)
})
})
c.Specify("correctly formats JSON output", func() {
config.Format = "json"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified", func() {
fileOutput.handleMessage(pack, &outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
})
c.Specify("and with a timestamp", func() {
fileOutput.prefix_ts = true
fileOutput.handleMessage(pack, &outData)
// Test will fail if date flips btn handleMessage call and
// todayStr calculation... should be extremely rare.
todayStr := time.Now().Format("[2006/Jan/02:")
strContents := toString(&outData)
msgJson, err := json.Marshal(pack.Message)
c.Assume(err, gs.IsNil)
c.Expect(strContents, pipeline_ts.StringContains, string(msgJson)+"\n")
c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
})
})
c.Specify("correctly formats protocol buffer stream output", func() {
config.Format = "protobufstream"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
outData := make([]byte, 0, 200)
c.Specify("when specified and timestamp ignored", func() {
fileOutput.prefix_ts = true
err := fileOutput.handleMessage(pack, &outData)
c.Expect(err, gs.IsNil)
b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
})
})
c.Specify("processes incoming messages", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Save for comparison.
payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
inChan <- pack
close(inChan)
outBatch := <-fileOutput.batchChan
wg.Wait()
c.Expect(string(outBatch), gs.Equals, payload)
})
c.Specify("Init halts if basedirectory is not writable", func() {
tmpdir := filepath.Join(os.TempDir(), "tmpdir")
err := os.MkdirAll(tmpdir, 0400)
c.Assume(err, gs.IsNil)
config.Path = tmpdir
err = fileOutput.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("commits to a file", func() {
outStr := "Write me out to the log file"
outBytes := []byte(outStr)
c.Specify("with default settings", func() {
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
contents, err := ioutil.ReadAll(tmpFile)
c.Assume(err, gs.IsNil)
c.Expect(string(contents), gs.Equals, outStr)
})
c.Specify("with different Perm settings", func() {
config.Perm = "600"
err := fileOutput.Init(config)
defer os.Remove(tmpFilePath)
c.Assume(err, gs.IsNil)
// Start committer loop
wg.Add(1)
go fileOutput.committer(oth.MockOutputRunner, &wg)
// Feed and close the batchChan
go func() {
fileOutput.batchChan <- outBytes
_ = <-fileOutput.backChan // clear backChan to prevent blocking
close(fileOutput.batchChan)
}()
wg.Wait()
// Wait for the file close operation to happen.
//for ; err == nil; _, err = fileOutput.file.Stat() {
//}
tmpFile, err := os.Open(tmpFilePath)
defer tmpFile.Close()
c.Assume(err, gs.IsNil)
fileInfo, err := tmpFile.Stat()
c.Assume(err, gs.IsNil)
fileMode := fileInfo.Mode()
if runtime.GOOS == "windows" {
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
} else {
// 7 consecutive dashes implies no perms for group or other
c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
}
})
})
c.Specify("honors folder_perm setting", func() {
config.FolderPerm = "750"
config.Path = filepath.Join(tmpFilePath, "subfile")
err := fileOutput.Init(config)
defer os.Remove(config.Path)
c.Assume(err, gs.IsNil)
fi, err := os.Stat(tmpFilePath)
c.Expect(fi.IsDir(), gs.IsTrue)
c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
})
c.Specify("that starts receiving w/ a flush interval", func() {
config.FlushInterval = 100000000 // We'll trigger the timer manually.
inChan := make(chan *PipelinePack)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
timerChan := make(chan time.Time)
msg2 := pipeline_ts.GetTestMessage()
pack2 := NewPipelinePack(pConfig.InputRecycleChan())
pack2.Message = msg2
recvWithConfig := func(config *FileOutputConfig) {
err := fileOutput.Init(config)
c.Assume(err, gs.IsNil)
wg.Add(1)
go fileOutput.receiver(oth.MockOutputRunner, &wg)
runtime.Gosched() // Yield so we can overwrite the timerChan.
fileOutput.timerChan = timerChan
}
cleanUp := func() {
close(inChan)
wg.Done()
}
c.Specify("honors flush interval", func() {
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case _ = <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case _ = <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval AND flush count", func() {
config.FlushCount = 2
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("honors flush interval OR flush count", func() {
config.FlushCount = 2
config.FlushOperator = "OR"
recvWithConfig(config)
defer cleanUp()
inChan <- pack
select {
case <-fileOutput.batchChan:
c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
default:
}
c.Specify("when interval triggers first", func() {
timerChan <- time.Now()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
c.Specify("when count triggers first", func() {
inChan <- pack2
runtime.Gosched()
select {
case <-fileOutput.batchChan:
default:
c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
}
})
})
})
})
}
func MatcherSpecificationSpec(c gospec.Context) {
msg := getTestMessage()
uuidStr := msg.GetUuidString()
data := []byte("data")
date := "Mon Jan 02 15:04:05 -0700 2006"
field1, _ := NewField("bytes", data, "")
field2, _ := NewField("int", int64(999), "")
field2.AddValue(int64(1024))
field3, _ := NewField("double", float64(99.9), "")
field4, _ := NewField("bool", true, "")
field5, _ := NewField("foo", "alternate", "")
field6, _ := NewField("Payload", "name=test;type=web;", "")
field7, _ := NewField("Timestamp", date, "date-time")
field8, _ := NewField("zero", int64(0), "")
field9, _ := NewField("string", "43", "")
msg.AddField(field1)
msg.AddField(field2)
msg.AddField(field3)
msg.AddField(field4)
msg.AddField(field5)
msg.AddField(field6)
msg.AddField(field7)
msg.AddField(field8)
msg.AddField(field9)
c.Specify("A MatcherSpecification", func() {
malformed := []string{
"",
"bogus",
"Type = 'test'", // invalid operator
"Pid == 'test='", // Pid is not a string
"Type == 'test' && (Severity==7 || Payload == 'Test Payload'", // missing paren
"Invalid == 'bogus'", // unknown variable name
"Fields[]", // empty name key
"Fields[test][]", // empty field index
"Fields[test][a]", // non numeric field index
"Fields[test][0][]", // empty array index
"Fields[test][0][a]", // non numeric array index
"Fields[test][0][0][]", // extra index dimension
"Fields[test][xxxx", // unmatched bracket
"Pid =~ /6/", // regex not allowed on numeric
"Pid !~ /6/", // regex not allowed on numeric
"Type =~ /test", // unmatched slash
"Type == /test/", // incorrect operator
"Type =~ 'test'", // string instead of regexp
"Type =~ /\\ytest/", // invalid escape character
"Type != 'test\"", // mis matched quote types
"Pid =~ 6", // number instead of regexp
"NIL", // invalid use of constant
"Type == NIL", // existence check only works on fields
"Fields[test] > NIL", // existence check only works with equals and not equals
}
negative := []string{
"FALSE",
"Type == 'test'&&(Severity==7||Payload=='Test Payload')",
"EnvVersion == '0.9'",
"EnvVersion != '0.8'",
"EnvVersion > '0.9'",
"EnvVersion >= '0.9'",
"EnvVersion < '0.8'",
"EnvVersion <= '0.7'",
"Severity == 5",
"Severity != 6",
"Severity < 6",
"Severity <= 5",
"Severity > 6",
"Severity >= 7",
"Fields[foo] == 'ba'",
"Fields[foo][1] == 'bar'",
"Fields[foo][0][1] == 'bar'",
"Fields[bool] == FALSE",
"Type =~ /Test/",
"Type !~ /TEST/",
"Payload =~ /^Payload/",
"Type == \"te'st\"",
"Type == 'te\"st'",
"Fields[int] =~ /999/",
"Fields[zero] == \"0\"",
"Fields[string] == 43",
"Fields[int] == NIL",
"Fields[int][0][1] == NIL",
"Fields[missing] != NIL",
"Type =~ /^te/",
"Type =~ /st$/",
"Type !~ /^TE/",
"Type !~ /ST$/",
"Logger =~ /./ && Type =~ /^anything/",
}
positive := []string{
"TRUE",
"(Severity == 7 || Payload == 'Test Payload') && Type == 'TEST'",
"EnvVersion == \"0.8\"",
"EnvVersion == '0.8'",
"EnvVersion != '0.9'",
"EnvVersion > '0.7'",
"EnvVersion >= '0.8'",
"EnvVersion < '0.9'",
"EnvVersion <= '0.8'",
"Hostname != ''",
"Logger == 'GoSpec'",
"Pid != 0",
"Severity != 5",
"Severity < 7",
"Severity <= 6",
"Severity == 6",
"Severity > 5",
"Severity >= 6",
"Timestamp > 0",
"Type != 'test'",
"Type == 'TEST' && Severity == 6",
"Type == 'test' && Severity == 7 || Payload == 'Test Payload'",
"Type == 'TEST'",
"Type == 'foo' || Type == 'bar' || Type == 'TEST'",
fmt.Sprintf("Uuid == '%s'", uuidStr),
"Fields[foo] == 'bar'",
"Fields[foo][0] == 'bar'",
"Fields[foo][0][0] == 'bar'",
"Fields[foo][1] == 'alternate'",
"Fields[foo][1][0] == 'alternate'",
"Fields[foo] == 'bar'",
"Fields[bytes] == 'data'",
"Fields[int] == 999",
"Fields[int][0][1] == 1024",
"Fields[double] == 99.9",
"Fields[bool] == TRUE",
"Type =~ /TEST/",
"Type !~ /bogus/",
"Type =~ /TEST/ && Payload =~ /Payload/",
"Fields[foo][1] =~ /alt/",
"Fields[Payload] =~ /name=\\w+/",
"Type =~ /(ST)/",
"Fields[int] != NIL",
"Fields[int][0][1] != NIL",
"Fields[int][0][2] == NIL",
"Fields[missing] == NIL",
"Type =~ /^TE/",
"Type =~ /ST$/",
"Type !~ /^te/",
"Type !~ /st$/",
}
c.Specify("malformed matcher tests", func() {
for _, v := range malformed {
_, err := CreateMatcherSpecification(v)
c.Expect(err, gs.Not(gs.IsNil))
}
})
c.Specify("negative matcher tests", func() {
for _, v := range negative {
ms, err := CreateMatcherSpecification(v)
c.Expect(err, gs.IsNil)
match := ms.Match(msg)
c.Expect(match, gs.IsFalse)
}
})
c.Specify("positive matcher tests", func() {
for _, v := range positive {
ms, err := CreateMatcherSpecification(v)
c.Expect(err, gs.IsNil)
match := ms.Match(msg)
c.Expect(match, gs.IsTrue)
}
})
})
}
func ReportSpec(c gs.Context) {
t := new(ts.SimpleT)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
chanSize := pConfig.Globals.PluginChanSize
checkForFields := func(c gs.Context, msg *message.Message) {
f0Val, ok := msg.GetFieldValue(f0.GetName())
c.Expect(ok, gs.IsTrue)
c.Expect(f0Val.(int64), gs.Equals, f0.GetValue().(int64))
f1Val, ok := msg.GetFieldValue(f1.GetName())
c.Expect(ok, gs.IsTrue)
c.Expect(f1Val.(string), gs.Equals, f1.GetValue().(string))
}
hasChannelData := func(msg *message.Message) (ok bool) {
capVal, _ := msg.GetFieldValue("InChanCapacity")
lenVal, _ := msg.GetFieldValue("InChanLength")
var i int64
if i, ok = capVal.(int64); !ok {
return
}
if ok = (i == int64(chanSize)); !ok {
return
}
if i, ok = lenVal.(int64); !ok {
return
}
ok = (i == int64(0))
return
}
fName := "counter"
filter := new(CounterFilter)
fRunner := NewFORunner(fName, filter, nil, chanSize)
var err error
fRunner.matcher, err = NewMatchRunner("Type == ''", "", fRunner, chanSize)
c.Assume(err, gs.IsNil)
fRunner.matcher.inChan = make(chan *PipelinePack, chanSize)
leakCount := 10
fRunner.SetLeakCount(leakCount)
iName := "stat_accum"
input := new(StatAccumInput)
iRunner := NewInputRunner(iName, input, nil, false)
c.Specify("`PopulateReportMsg`", func() {
msg := ts.GetTestMessage()
c.Specify("w/ a filter", func() {
err := PopulateReportMsg(fRunner, msg)
c.Assume(err, gs.IsNil)
c.Specify("invokes `ReportMsg` on the filter", func() {
checkForFields(c, msg)
})
c.Specify("adds the channel data", func() {
c.Expect(hasChannelData(msg), gs.IsTrue)
})
c.Specify("has its leak count set properly", func() {
leakVal, ok := msg.GetFieldValue("LeakCount")
c.Assume(ok, gs.IsTrue)
i, ok := leakVal.(int64)
c.Assume(ok, gs.IsTrue)
c.Expect(int(i), gs.Equals, leakCount)
})
})
c.Specify("w/ an input", func() {
err := PopulateReportMsg(iRunner, msg)
c.Assume(err, gs.IsNil)
c.Specify("invokes `ReportMsg` on the input", func() {
checkForFields(c, msg)
})
c.Specify("doesn't add any channel data", func() {
capVal, ok := msg.GetFieldValue("InChanCapacity")
c.Expect(capVal, gs.IsNil)
c.Expect(ok, gs.IsFalse)
lenVal, ok := msg.GetFieldValue("InChanLength")
c.Expect(lenVal, gs.IsNil)
c.Expect(ok, gs.IsFalse)
})
})
})
c.Specify("PipelineConfig", func() {
pc := NewPipelineConfig(nil)
// Initialize all of the PipelinePacks that we'll need
pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)
pc.FilterRunners = map[string]FilterRunner{fName: fRunner}
pc.InputRunners = map[string]InputRunner{iName: iRunner}
c.Specify("returns full set of accurate reports", func() {
reportChan := make(chan *PipelinePack)
go pc.reports(reportChan)
reports := make(map[string]*PipelinePack)
for r := range reportChan {
iName, ok := r.Message.GetFieldValue("name")
c.Expect(ok, gs.IsTrue)
name, ok := iName.(string)
c.Expect(ok, gs.IsTrue)
c.Expect(name, gs.Not(gs.Equals), "MISSING")
reports[name] = r
pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)
}
fReport := reports[fName]
c.Expect(fReport, gs.Not(gs.IsNil))
checkForFields(c, fReport.Message)
c.Expect(hasChannelData(fReport.Message), gs.IsTrue)
iReport := reports[iName]
c.Expect(iReport, gs.Not(gs.IsNil))
checkForFields(c, iReport.Message)
recycleReport := reports["inputRecycleChan"]
c.Expect(recycleReport, gs.Not(gs.IsNil))
capVal, ok := recycleReport.Message.GetFieldValue("InChanCapacity")
c.Expect(ok, gs.IsTrue)
c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))
injectReport := reports["injectRecycleChan"]
c.Expect(injectReport, gs.Not(gs.IsNil))
capVal, ok = injectReport.Message.GetFieldValue("InChanCapacity")
c.Expect(ok, gs.IsTrue)
c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))
routerReport := reports["Router"]
c.Expect(routerReport, gs.Not(gs.IsNil))
c.Expect(hasChannelData(routerReport.Message), gs.IsTrue)
})
})
}
func StatAccumInputSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(InputTestHelper)
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
c.Specify("A StatAccumInput using normal namespaces", func() {
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Inject(ith.Pack)
ith.MockInputRunner.EXPECT().Ticker()
statAccumInput := StatAccumInput{}
config := statAccumInput.ConfigStruct().(*StatAccumInputConfig)
config.EmitInFields = true
config.EmitInPayload = false
err := statAccumInput.Init(config)
c.Expect(err, gs.IsNil)
var wg sync.WaitGroup
prepareSendingStats := func() {
wg.Add(1)
go func() {
err := statAccumInput.Run(ith.MockInputRunner, ith.MockHelper)
wg.Done()
c.Expect(err, gs.IsNil)
}()
}
finalizeSendingStats := func() *message.Message {
close(statAccumInput.statChan)
wg.Wait()
return ith.Pack.Message
}
sendTimer := func(key string, vals ...int) {
for _, v := range vals {
statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "ms", float32(1)}
}
}
sendCounter := func(key string, vals ...int) {
for _, v := range vals {
statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "c", float32(1)}
}
}
sendGauge := func(key string, vals ...int) {
for _, v := range vals {
statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "g", float32(1)}
}
}
validateValueAtKey := func(msg *message.Message, key string, value interface{}) {
fieldValue, ok := msg.GetFieldValue(key)
c.Expect(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, value)
}
c.Specify("emits timer with correct prefixes", func() {
prepareSendingStats()
sendTimer("sample.timer", 10, 10, 20, 20)
sendTimer("sample2.timer", 10, 20)
msg := finalizeSendingStats()
validateValueAtKey(msg, "sample.timer.count", int64(4))
validateValueAtKey(msg, "sample.timer.mean", 15.0)
validateValueAtKey(msg, "sample.timer.lower", 10.0)
validateValueAtKey(msg, "sample2.timer.count", int64(2))
validateValueAtKey(msg, "sample2.timer.mean", 15.0)
validateValueAtKey(msg, "sample2.timer.lower", 10.0)
validateValueAtKey(msg, "statsd.numStats", int64(2))
})
c.Specify("emits counters with correct prefixes", func() {
prepareSendingStats()
sendCounter("sample.cnt", 1, 2, 3, 4, 5)
sendCounter("sample2.cnt", 159, 951)
msg := finalizeSendingStats()
validateValueAtKey(msg, "sample.cnt.count", int64(15))
validateValueAtKey(msg, "sample.cnt.rate", 1.5)
validateValueAtKey(msg, "sample2.cnt.count", int64(1110))
validateValueAtKey(msg, "sample2.cnt.rate", 1110.0/float64(config.TickerInterval))
})
c.Specify("emits gauge with correct prefixes", func() {
prepareSendingStats()
sendGauge("sample.gauge", 1, 2)
sendGauge("sample2.gauge", 1, 2, 3, 4, 5)
msg := finalizeSendingStats()
validateValueAtKey(msg, "sample.gauge", int64(2))
validateValueAtKey(msg, "sample2.gauge", int64(5))
})
c.Specify("emits correct statsd.numStats count", func() {
prepareSendingStats()
sendGauge("sample.gauge", 1, 2)
sendGauge("sample2.gauge", 1, 2)
sendCounter("sample.cnt", 1, 2, 3, 4, 5)
sendCounter("sample2.cnt", 159, 951)
sendTimer("sample.timer", 10, 10, 20, 20)
sendTimer("sample2.timer", 10, 20)
msg := finalizeSendingStats()
validateValueAtKey(msg, "statsd.numStats", int64(6))
})
})
c.Specify("A StatAccumInput using Legacy namespaces", func() {
statAccumInput := StatAccumInput{}
config := statAccumInput.ConfigStruct().(*StatAccumInputConfig)
config.LegacyNamespaces = true
tickChan := make(chan time.Time)
c.Specify("must emit data in payload and/or message fields", func() {
config.EmitInPayload = false
err := statAccumInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
expected := "One of either `EmitInPayload` or `EmitInFields` must be set to true."
c.Expect(err.Error(), gs.Equals, expected)
})
c.Specify("that actually emits a message", func() {
statName := "sample.stat"
statVal := int64(303)
testStat := Stat{statName, strconv.Itoa(int(statVal)), "c", float32(1)}
validateMsgFields := func(msg *message.Message) {
c.Expect(len(msg.Fields), gs.Equals, 4)
// timestamp
_, ok := msg.GetFieldValue("timestamp")
c.Expect(ok, gs.IsTrue)
var tmp interface{}
var intTmp int64
// stats.sample.stat
tmp, ok = msg.GetFieldValue("stats." + statName)
c.Expect(ok, gs.IsTrue)
intTmp, ok = tmp.(int64)
c.Expect(ok, gs.IsTrue)
c.Expect(intTmp, gs.Equals, int64(30))
// stats_counts.sample.stat
tmp, ok = msg.GetFieldValue("stats_counts." + statName)
c.Expect(ok, gs.IsTrue)
intTmp, ok = tmp.(int64)
c.Expect(ok, gs.IsTrue)
c.Expect(intTmp, gs.Equals, statVal)
// statsd.numStats
tmp, ok = msg.GetFieldValue("statsd.numStats")
c.Expect(ok, gs.IsTrue)
intTmp, ok = tmp.(int64)
c.Expect(ok, gs.IsTrue)
c.Expect(intTmp, gs.Equals, int64(1))
}
validateMsgPayload := func(msg *message.Message) {
lines := strings.Split(msg.GetPayload(), "\n")
c.Expect(len(lines), gs.Equals, 4)
c.Expect(lines[3], gs.Equals, "")
var timestamp string
for i := 0; i < 3; i++ {
line := strings.Split(lines[i], " ")
c.Expect(len(line), gs.Equals, 3)
switch i {
case 0:
c.Expect(line[0], gs.Equals, "stats."+statName)
c.Expect(line[1], gs.Equals, "30.300000")
timestamp = line[2]
case 1:
c.Expect(line[0], gs.Equals, "stats_counts."+statName)
c.Expect(line[1], gs.Equals, strconv.Itoa(int(statVal)))
c.Expect(line[2], gs.Equals, timestamp)
case 2:
c.Expect(line[0], gs.Equals, "statsd.numStats")
c.Expect(line[1], gs.Equals, "1")
c.Expect(line[2], gs.Equals, timestamp)
}
}
expected := strings.Join(lines, "\n")
c.Expect(msg.GetPayload(), gs.Equals, expected)
}
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Inject(ith.Pack)
ith.MockInputRunner.EXPECT().Ticker()
var wg sync.WaitGroup
startAndSwapTickChan := func() {
wg.Add(1)
go func() {
err := statAccumInput.Run(ith.MockInputRunner, ith.MockHelper)
wg.Done()
c.Expect(err, gs.IsNil)
}()
time.Sleep(50) // Kludgey wait for tickChan to be set so we can replace.
statAccumInput.tickChan = tickChan
}
c.Specify("emits data in payload by default", func() {
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startAndSwapTickChan()
statAccumInput.statChan <- testStat
close(statAccumInput.statChan)
wg.Wait()
validateMsgPayload(ith.Pack.Message)
})
c.Specify("emits data in fields when specified", func() {
config.EmitInFields = true
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startAndSwapTickChan()
statAccumInput.statChan <- testStat
close(statAccumInput.statChan)
wg.Wait()
validateMsgFields(ith.Pack.Message)
validateMsgPayload(ith.Pack.Message)
})
c.Specify("omits data in payload when specified", func() {
config.EmitInPayload = false
config.EmitInFields = true
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startAndSwapTickChan()
statAccumInput.statChan <- testStat
close(statAccumInput.statChan)
wg.Wait()
validateMsgFields(ith.Pack.Message)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "")
})
c.Specify("honors time ticker to flush", func() {
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startAndSwapTickChan()
statAccumInput.statChan <- testStat
tickChan <- time.Now()
validateMsgPayload(ith.Pack.Message)
ith.PackSupply <- ith.Pack
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Inject(ith.Pack)
close(statAccumInput.statChan)
wg.Wait()
})
c.Specify("correctly processes timers", func() {
sendTimer := func(vals ...int) {
for _, v := range vals {
statAccumInput.statChan <- Stat{"sample.timer", strconv.Itoa(int(v)), "ms", float32(1)}
}
}
config.EmitInFields = true
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startAndSwapTickChan()
sendTimer(220, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100)
close(statAccumInput.statChan)
wg.Wait()
msg := ith.Pack.Message
getVal := func(token string) float64 {
tmp, ok := msg.GetFieldValue("stats.timers.sample.timer." + token)
c.Expect(ok, gs.IsTrue)
val, ok := tmp.(float64)
c.Expect(ok, gs.IsTrue)
return val
}
c.Expect(getVal("upper"), gs.Equals, 220.0)
c.Expect(getVal("lower"), gs.Equals, 10.0)
c.Expect(getVal("mean"), gs.Equals, 70.0)
c.Expect(getVal("upper_90"), gs.Equals, 100.0)
c.Expect(getVal("mean_90"), gs.Equals, 55.0)
tmp, ok := msg.GetFieldValue("stats.timers.sample.timer.count")
c.Expect(ok, gs.IsTrue)
intTmp, ok := tmp.(int64)
c.Expect(ok, gs.IsTrue)
c.Expect(intTmp, gs.Equals, int64(11))
})
})
})
}
func TcpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
key := "testkey"
signers := map[string]Signer{"test_1": {key}}
signer := "test"
c.Specify("A TcpInput protobuf parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
Signers: signers,
Decoder: "ProtobufDecoder",
ParserType: "message.proto"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr)
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
enccall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "TcpInput-123-ProtobufDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
cleanup := func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}
c.Specify("reads a message from its connection", func() {
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
})
c.Specify("reads a MD5 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a SHA1 signed message from its connection", func() {
header.SetHmacHashFunction(message.Header_SHA1)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(sha1.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-ith.DecodeChan:
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Signer, gs.Equals, "test")
case t := <-timeout:
c.Expect(t, gs.IsNil)
}
})
c.Specify("reads a signed message with an expired key from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(11)) // non-existent key version
hm := hmac.New(md5.New, []byte(key))
hm.Write(mbytes)
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
header.SetHmacHashFunction(message.Header_MD5)
header.SetHmacSigner(signer)
header.SetHmacKeyVersion(uint32(1))
hm := hmac.New(md5.New, []byte(key))
hm.Write([]byte("some bytes"))
header.SetHmac(hm.Sum(nil))
hbytes, _ := proto.Marshal(header)
buflen := 3 + len(hbytes) + len(mbytes)
readCall.Return(buflen, nil)
readCall.Do(getPayloadBytes(hbytes, mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer cleanup()
ith.PackSupply <- ith.Pack
timeout := make(chan bool, 1)
go func() {
time.Sleep(100 * time.Millisecond)
timeout <- true
}()
select {
case packRef := <-mockDecoderRunner.InChan():
c.Expect(packRef, gs.IsNil)
case t := <-timeout:
c.Expect(t, gs.IsTrue)
}
})
})
c.Specify("A TcpInput regexp parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
Decoder: "RegexpDecoder",
ParserType: "regexp",
}
tcpInput := TcpInput{}
err := tcpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
mbytes := []byte("this is a test message\n")
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("logger")
enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "TcpInput-123-RegexpDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
c.Specify("reads a message from its connection", func() {
readCall.Return(len(mbytes), nil)
readCall.Do(getPayloadText(mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes[:len(mbytes)-1]))
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
})
})
c.Specify("A TcpInput token parser", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
tcpInput := TcpInput{}
err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
Decoder: "TokenDecoder",
ParserType: "token",
Delimiter: "\n"})
c.Assume(err, gs.IsNil)
realListener := tcpInput.listener
c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
realListener.Close()
mockConnection := pipeline_ts.NewMockConn(ctrl)
mockListener := pipeline_ts.NewMockListener(ctrl)
tcpInput.listener = mockListener
addr := new(address)
addr.str = "123"
mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
mbytes := []byte("this is a test message\n")
err = errors.New("connection closed") // used in the read return(s)
readCall := mockConnection.EXPECT().Read(gomock.Any())
readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
readEnd.Return(0, err)
mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
mockConnection.EXPECT().Close()
neterr := pipeline_ts.NewMockError(ctrl)
neterr.EXPECT().Temporary().Return(false)
acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
acceptCall.Do(func() {
acceptCall = mockListener.EXPECT().Accept()
acceptCall.Return(nil, neterr)
})
mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
ith.MockInputRunner.EXPECT().Name().Return("logger")
enccall := ith.MockHelper.EXPECT().DecoderRunner("TokenDecoder", "TcpInput-123-TokenDecoder").AnyTimes()
enccall.Return(ith.Decoder, true)
ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)
c.Specify("reads a message from its connection", func() {
readCall.Return(len(mbytes), nil)
readCall.Do(getPayloadText(mbytes))
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
mockListener.EXPECT().Close()
tcpInput.Stop()
tcpInput.wg.Wait()
}()
ith.PackSupply <- ith.Pack
packRef := <-ith.DecodeChan
c.Expect(ith.Pack, gs.Equals, packRef)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes))
c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
})
})
c.Specify("A TcpInput using TLS", func() {
tcpInput := TcpInput{}
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
ParserType: "token",
UseTls: true,
}
c.Specify("fails to init w/ missing key or cert file", func() {
config.Tls = TlsConfig{}
err := tcpInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
})
c.Specify("accepts TLS client connections", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
}
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
tcpInput.Stop()
tcpInput.wg.Wait()
}()
clientConfig := new(tls.Config)
clientConfig.InsecureSkipVerify = true
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(err, gs.IsNil)
defer conn.Close()
conn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
n, err := conn.Write([]byte("This is a test."))
c.Expect(err, gs.IsNil)
c.Expect(n, gs.Equals, len("This is a test."))
})
c.Specify("doesn't accept connections below specified min TLS version", func() {
ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
MinVersion: "TLS12",
}
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
tcpInput.Stop()
tcpInput.wg.Wait()
time.Sleep(time.Duration(1000))
}()
clientConfig := &tls.Config{
InsecureSkipVerify: true,
MaxVersion: tls.VersionTLS11,
}
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(conn, gs.IsNil)
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func ProcessDirectoryInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
err := pConfig.RegisterDefault("NullSplitter")
c.Assume(err, gs.IsNil)
c.Specify("A ProcessDirectoryInput", func() {
pdiInput := ProcessDirectoryInput{}
pdiInput.SetPipelineConfig(pConfig)
config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig)
workingDir, err := os.Getwd()
c.Assume(err, gs.IsNil)
config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes")
// `Ticker` is the last thing called during the setup part of the
// input's `Run` method, so it triggers a waitgroup that tests can
// wait on when they need to ensure initialization has finished.
var started sync.WaitGroup
started.Add(1)
tickChan := make(chan time.Time, 1)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do(
func() {
started.Done()
})
// Similarly we use a waitgroup to signal when LogMessage has been
// called to know when reloads have completed. Warning: If you call
// expectLogMessage with a msg that is never passed to LogMessage and
// then you call loaded.Wait() then your test will hang and never
// complete.
var loaded sync.WaitGroup
expectLogMessage := func(msg string) {
loaded.Add(1)
ith.MockInputRunner.EXPECT().LogMessage(msg).Do(
func(msg string) {
loaded.Done()
})
}
// Same name => same content.
paths := []string{
filepath.Join(config.ProcessDir, "100", "h0.toml"),
filepath.Join(config.ProcessDir, "100", "h1.toml"),
filepath.Join(config.ProcessDir, "200", "h0.toml"),
filepath.Join(config.ProcessDir, "300", "h1.toml"),
}
copyFile := func(src, dest string) {
inFile, err := os.Open(src)
c.Assume(err, gs.IsNil)
outFile, err := os.Create(dest)
c.Assume(err, gs.IsNil)
_, err = io.Copy(outFile, inFile)
c.Assume(err, gs.IsNil)
inFile.Close()
outFile.Close()
}
err = pdiInput.Init(config)
c.Expect(err, gs.IsNil)
for _, p := range paths {
expectLogMessage("Added: " + p)
}
go pdiInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
pdiInput.Stop()
for _, entry := range pdiInput.inputs {
entry.ir.Input().Stop()
}
}()
started.Wait()
c.Specify("loads scheduled jobs", func() {
pathIndex := func(name string) (i int) {
var p string
for i, p = range paths {
if name == p {
return
}
}
return -1
}
for name, entry := range pdiInput.inputs {
i := pathIndex(name)
// Make sure each file path got registered.
c.Expect(i, gs.Not(gs.Equals), -1)
dirName := filepath.Base(filepath.Dir(name))
dirInt, err := strconv.Atoi(dirName)
c.Expect(err, gs.IsNil)
// And that the ticker interval was read correctly.
c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval)
}
})
c.Specify("discovers and adds a new job", func() {
// Copy one of the files to register a new process.
newPath := filepath.Join(config.ProcessDir, "300", "h0.toml")
copyFile(paths[0], newPath)
defer func() {
err := os.Remove(newPath)
c.Assume(err, gs.IsNil)
}()
// Set up expectations and trigger process dir reload.
expectLogMessage("Added: " + newPath)
tickChan <- time.Now()
loaded.Wait()
// Make sure our plugin was loaded.
c.Expect(len(pdiInput.inputs), gs.Equals, 5)
newEntry, ok := pdiInput.inputs[newPath]
c.Expect(ok, gs.IsTrue)
c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300))
})
c.Specify("removes a deleted job", func() {
err := os.Remove(paths[3])
c.Assume(err, gs.IsNil)
defer func() {
copyFile(paths[1], paths[3])
}()
// Set up expectations and trigger process dir reload.
expectLogMessage("Removed: " + paths[3])
tickChan <- time.Now()
loaded.Wait()
// Make sure our plugin was deleted.
c.Expect(len(pdiInput.inputs), gs.Equals, 3)
})
c.Specify("notices a changed job", func() {
// Overwrite one job w/ a slightly different one.
copyFile(paths[0], paths[3])
defer copyFile(paths[1], paths[3])
// Set up expectations and trigger process dir reload.
expectLogMessage("Removed: " + paths[3])
expectLogMessage("Added: " + paths[3])
tickChan <- time.Now()
loaded.Wait()
// Make sure the new config was loaded.
c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals,
"hello world\n")
})
})
}
func ProcessChainSpec(c gs.Context) {
readCommandOutput := func(reader io.Reader, resultChan chan string) {
data, err := ioutil.ReadAll(reader)
c.Expect(err, gs.IsNil)
resultChan <- string(data)
}
c.Specify("A ManagedCommand", func() {
c.Specify("can run a single command", func() {
Path := SINGLE_CMD
cmd := NewManagedCmd(Path, SINGLE_CMD_ARGS, 0)
output := make(chan string)
cmd.Start(true)
go readCommandOutput(cmd.Stdout_r, output)
cmd.Wait()
outputStr := <-output
c.Expect(fmt.Sprintf("%x", outputStr), gs.Equals,
fmt.Sprintf("%x", SINGLE_CMD_OUTPUT))
})
c.Specify("honors nonzero timeouts", func() {
// Timeout should always occur inside of 10 seconds.
Path := NONZERO_TIMEOUT_CMD
timeout := NONZERO_TIMEOUT
cmd := NewManagedCmd(Path, NONZERO_TIMEOUT_ARGS, timeout)
output := make(chan string)
cmd.Start(true)
start := time.Now()
go readCommandOutput(cmd.Stdout_r, output)
cmd.Wait()
end := time.Now()
<-output
actualDuration := end.Sub(start)
c.Expect(actualDuration < time.Second*10, gs.Equals, true)
})
c.Specify("reads process stderr properly", func() {
Path := STDERR_CMD
cmd := NewManagedCmd(Path, STDERR_CMD_ARGS, 0)
stdoutResults := make(chan string, 1)
stderrResults := make(chan string, 1)
cmd.Start(true)
go readCommandOutput(cmd.Stdout_r, stdoutResults)
go readCommandOutput(cmd.Stderr_r, stderrResults)
cmd.Wait()
// Error messages will vary platform to platform, just check that
// there is some message which will be about "No such file found".
c.Expect(len(<-stderrResults) > 0, gs.Equals, true)
c.Expect(<-stdoutResults, gs.Equals, "")
})
c.Specify("can be terminated before timeout occurs", func() {
Path := NONZERO_TIMEOUT_CMD
timeout := time.Second * 30
cmd := NewManagedCmd(Path, NONZERO_TIMEOUT_ARGS, timeout)
stdoutResults := make(chan string, 1)
cmd.Start(true)
start := time.Now()
go readCommandOutput(cmd.Stdout_r, stdoutResults)
time.Sleep(NONZERO_TIMEOUT)
cmd.Stopchan <- true
cmd.Wait()
end := time.Now()
actualDuration := end.Sub(start)
c.Expect(actualDuration < timeout, gs.Equals, true)
})
c.Specify("can reset commands to run again", func() {
Path := SINGLE_CMD
cmd := NewManagedCmd(Path, SINGLE_CMD_ARGS, 0)
stdoutResults := make(chan string, 1)
cmd.Start(true)
go readCommandOutput(cmd.Stdout_r, stdoutResults)
cmd.Wait()
c.Expect(<-stdoutResults, gs.Equals, SINGLE_CMD_OUTPUT)
// Reset and rerun it.
cmd = cmd.clone()
cmd.Start(true)
go readCommandOutput(cmd.Stdout_r, stdoutResults)
cmd.Wait()
c.Expect(<-stdoutResults, gs.Equals, SINGLE_CMD_OUTPUT)
})
})
c.Specify("A New ProcessChain", func() {
c.Specify("pipes multiple commands correctly", func() {
chain := NewCommandChain(0)
chain.AddStep(PIPE_CMD1, PIPE_CMD1_ARGS...)
chain.AddStep(PIPE_CMD2, PIPE_CMD2_ARGS...)
err := chain.Start()
c.Expect(err, gs.IsNil)
stdoutReader, err := chain.Stdout_r()
c.Expect(err, gs.IsNil)
stdoutResult := make(chan string, 1)
stderrReader, err := chain.Stderr_r()
c.Expect(err, gs.IsNil)
stderrResult := make(chan string, 1)
go readCommandOutput(stdoutReader, stdoutResult)
go readCommandOutput(stderrReader, stderrResult)
err = chain.Wait()
c.Expect(err, gs.IsNil)
c.Expect(<-stderrResult, gs.Equals, "")
c.Expect(<-stdoutResult, gs.Equals, PIPE_CMD_OUTPUT)
})
c.Specify("will honor timeouts", func() {
// This test must timeout within 10 seconds.
var err error
timeout := NONZERO_TIMEOUT
chain := NewCommandChain(timeout)
chain.AddStep(TIMEOUT_PIPE_CMD1, TIMEOUT_PIPE_CMD1_ARGS...)
chain.AddStep(TIMEOUT_PIPE_CMD2, TIMEOUT_PIPE_CMD2_ARGS...)
err = chain.Start()
start := time.Now()
c.Expect(err, gs.IsNil)
err = chain.Wait()
end := time.Now()
actual_duration := end.Sub(start)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(strings.Contains(err.Error(), "was killed"), gs.Equals, true)
c.Expect(actual_duration < time.Second*10, gs.Equals, true)
})
c.Specify("will stop chains before timeout has completed", func() {
var err error
timeout := time.Second * 30
chain := NewCommandChain(timeout)
chain.AddStep(TIMEOUT_PIPE_CMD1, TIMEOUT_PIPE_CMD1_ARGS...)
chain.AddStep(TIMEOUT_PIPE_CMD2, TIMEOUT_PIPE_CMD2_ARGS...)
err = chain.Start()
start := time.Now()
c.Expect(err, gs.IsNil)
time.Sleep(NONZERO_TIMEOUT)
chain.Stopchan <- true
err = chain.Wait()
end := time.Now()
actual_duration := end.Sub(start)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(actual_duration < timeout, gs.Equals, true)
})
c.Specify("can reset chains to run again", func() {
// This test assumes tail and grep
var err error
chain := NewCommandChain(0)
chain.AddStep(PIPE_CMD1, PIPE_CMD1_ARGS...)
chain.AddStep(PIPE_CMD2, PIPE_CMD2_ARGS...)
err = chain.Start()
c.Expect(err, gs.IsNil)
stdoutReader, err := chain.Stdout_r()
c.Expect(err, gs.IsNil)
stdoutResult := make(chan string, 1)
stderrReader, err := chain.Stderr_r()
c.Expect(err, gs.IsNil)
stderrResult := make(chan string, 1)
go readCommandOutput(stdoutReader, stdoutResult)
go readCommandOutput(stderrReader, stderrResult)
err = chain.Wait()
c.Expect(err, gs.IsNil)
c.Expect(<-stderrResult, gs.Equals, "")
c.Expect(<-stdoutResult, gs.Equals, PIPE_CMD_OUTPUT)
// Reset the chain for a second run
chain = chain.clone()
err = chain.Start()
c.Expect(err, gs.IsNil)
stdoutReader, err = chain.Stdout_r()
c.Expect(err, gs.IsNil)
stderrReader, err = chain.Stderr_r()
c.Expect(err, gs.IsNil)
go readCommandOutput(stdoutReader, stdoutResult)
go readCommandOutput(stderrReader, stderrResult)
err = chain.Wait()
c.Expect(err, gs.IsNil)
c.Expect(<-stderrResult, gs.Equals, "")
c.Expect(<-stdoutResult, gs.Equals, PIPE_CMD_OUTPUT)
})
})
}
func LogstreamerInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
here, _ := os.Getwd()
dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
c.Expect(tmpErr, gs.Equals, nil)
defer func() {
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
config := NewPipelineConfig(globals)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Specify localhost, but we're not really going to use the network
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.DecodeChan = make(chan *PipelinePack)
c.Specify("A LogstreamerInput", func() {
lsInput := &LogstreamerInput{pConfig: config}
lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
lsiConfig.LogDirectory = dirPath
lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
lsiConfig.Differentiator = []string{"logfile"}
lsiConfig.Priority = []string{"^Seq"}
lsiConfig.Decoder = "decoder-name"
c.Specify("w/ no translation map", func() {
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
c.Expect(len(lsInput.plugins), gs.Equals, 1)
mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
// Create pool of packs.
numLines := 5 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
// Expect calls to get decoder and decode each message. Since the
// decoding is a no-op, the message payload will be the log file
// line, unchanged.
pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
"-"+lsiConfig.Decoder)
pbcall.Return(mockDecoderRunner, true)
decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
decodeCall.Return(ith.DecodeChan)
runOutChan := make(chan error, 1)
go func() {
err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
runOutChan <- err
}()
d, _ := time.ParseDuration("5s")
timeout := time.After(d)
timed := false
for x := 0; x < numLines; x++ {
select {
case <-ith.DecodeChan:
case <-timeout:
timed = true
x += numLines
}
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lsInput.Stop()
c.Expect(timed, gs.Equals, false)
c.Expect(<-runOutChan, gs.Equals, nil)
})
})
c.Specify("with a translation map", func() {
lsiConfig.Translation = make(ls.SubmatchTranslationMap)
lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)
c.Specify("allows len 1 translation map for 'missing'", func() {
lsiConfig.Translation["Seq"]["missing"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
})
c.Specify("doesn't allow len 1 map for other keys", func() {
lsiConfig.Translation["Seq"]["missin"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"A translation map with one entry ('Seq') must be specifying a "+
"'missing' key.")
})
})
})
}
func ESEncodersSpec(c gs.Context) {
recycleChan := make(chan *PipelinePack, 1)
pack := NewPipelinePack(recycleChan)
pack.Message = getTestMessageWithFunnyFields()
c.Specify("writeStringField", func() {
buf := bytes.Buffer{}
c.Specify("should properly encode special characters in json", func() {
writeStringField(true, &buf, `hello"bar`, "world\nfoo\\")
c.Expect(buf.String(), gs.Equals, `"hello\u0022bar":"world\u000afoo\u005c"`)
})
c.Specify("Should replace invalid utf8 with replacement character", func() {
writeStringField(true, &buf, "\xa3", "\xa3")
c.Expect(buf.String(), gs.Equals, "\"\xEF\xBF\xBD\":\"\xEF\xBF\xBD\"")
})
})
c.Specify("interpolateFlag", func() {
c.Specify("should interpolate for index and type names", func() {
interpolatedIndex, err := interpolateFlag(&ElasticSearchCoordinates{},
pack.Message, "heka-%{Pid}-%{\"foo}-%{%Y.%m.%d}")
c.Expect(err, gs.IsNil)
t := time.Now().UTC()
c.Expect(interpolatedIndex, gs.Equals, "heka-14098-bar\n-"+t.Format("2006.01.02"))
interpolatedType, err := interpolateFlag(&ElasticSearchCoordinates{},
pack.Message, "%{Type}")
c.Expect(err, gs.IsNil)
c.Expect(interpolatedType, gs.Equals, "TEST")
})
c.Specify("should interpolate from message field", func() {
id := "%{idField}"
interpolatedId, err := interpolateFlag(&ElasticSearchCoordinates{},
pack.Message, id)
c.Expect(err, gs.IsNil)
c.Expect(interpolatedId, gs.Equals, "1234")
})
c.Specify("should fail for nonexistent message field", func() {
id := "%{idFail}"
unInterpolatedId, err := interpolateFlag(&ElasticSearchCoordinates{},
pack.Message, id)
c.Expect(strings.Contains(err.Error(),
"Could not interpolate field from config: %{idFail}"), gs.IsTrue)
c.Expect(unInterpolatedId, gs.Equals, "idFail")
})
})
c.Specify("ESLogstashV0Encoder", func() {
encoder := new(ESLogstashV0Encoder)
config := encoder.ConfigStruct().(*ESLogstashV0EncoderConfig)
config.RawBytesFields = []string{
"test_raw_field_string",
"test_raw_field_bytes",
"test_raw_field_string_array",
"test_raw_field_bytes_array",
}
c.Specify("Should properly encode a message", func() {
err := encoder.Init(config)
c.Assume(err, gs.IsNil)
b, err := encoder.Encode(pack)
c.Assume(err, gs.IsNil)
output := string(b)
lines := strings.Split(output, string(NEWLINE))
decoded := make(map[string]interface{})
err = json.Unmarshal([]byte(lines[0]), &decoded)
c.Assume(err, gs.IsNil)
sub := decoded["index"].(map[string]interface{})
t := time.Now().UTC()
c.Expect(sub["_index"], gs.Equals, "logstash-"+t.Format("2006.01.02"))
c.Expect(sub["_type"], gs.Equals, "message")
decoded = make(map[string]interface{})
err = json.Unmarshal([]byte(lines[1]), &decoded)
c.Assume(err, gs.IsNil)
c.Expect(decoded["@fields"].(map[string]interface{})[`"foo`], gs.Equals, "bar\n")
c.Expect(decoded["@fields"].(map[string]interface{})[`"number`], gs.Equals, 64.0)
c.Expect(decoded["@fields"].(map[string]interface{})["\xEF\xBF\xBD"], gs.Equals,
"\xEF\xBF\xBD")
c.Expect(decoded["@uuid"], gs.Equals, "87cf1ac2-e810-4ddf-a02d-a5ce44d13a85")
c.Expect(decoded["@timestamp"], gs.Equals, "2013-07-16T15:49:05")
c.Expect(decoded["@type"], gs.Equals, "message")
c.Expect(decoded["@logger"], gs.Equals, "GoSpec")
c.Expect(decoded["@severity"], gs.Equals, 6.0)
c.Expect(decoded["@message"], gs.Equals, "Test Payload")
c.Expect(decoded["@envversion"], gs.Equals, "0.8")
c.Expect(decoded["@pid"], gs.Equals, 14098.0)
c.Expect(decoded["@source_host"], gs.Equals, "hostname")
stringsArray := decoded["@fields"].(map[string]interface{})["stringArray"].([]interface{})
c.Expect(len(stringsArray), gs.Equals, 4)
c.Expect(stringsArray[0], gs.Equals, "asdf")
c.Expect(stringsArray[1], gs.Equals, "jkl;")
c.Expect(stringsArray[2], gs.Equals, "push")
c.Expect(stringsArray[3], gs.Equals, "pull")
bytesArray := decoded["@fields"].(map[string]interface{})["byteArray"].([]interface{})
c.Expect(len(bytesArray), gs.Equals, 2)
c.Expect(bytesArray[0], gs.Equals, base64.StdEncoding.EncodeToString([]byte("asdf")))
c.Expect(bytesArray[1], gs.Equals, base64.StdEncoding.EncodeToString([]byte("jkl;")))
integerArray := decoded["@fields"].(map[string]interface{})["integerArray"].([]interface{})
c.Expect(len(integerArray), gs.Equals, 3)
c.Expect(integerArray[0], gs.Equals, 22.0)
c.Expect(integerArray[1], gs.Equals, 80.0)
c.Expect(integerArray[2], gs.Equals, 3000.0)
doubleArray := decoded["@fields"].(map[string]interface{})["doubleArray"].([]interface{})
c.Expect(len(doubleArray), gs.Equals, 2)
c.Expect(doubleArray[0], gs.Equals, 42.0)
c.Expect(doubleArray[1], gs.Equals, 19101.3)
boolArray := decoded["@fields"].(map[string]interface{})["boolArray"].([]interface{})
c.Expect(len(boolArray), gs.Equals, 5)
c.Expect(boolArray[0], gs.Equals, true)
c.Expect(boolArray[1], gs.Equals, false)
c.Expect(boolArray[2], gs.Equals, false)
c.Expect(boolArray[3], gs.Equals, false)
c.Expect(boolArray[4], gs.Equals, true)
c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_string"].(map[string]interface{})["asdf"], gs.Equals, 123.0)
c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_bytes"].(map[string]interface{})["asdf"], gs.Equals, 123.0)
c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_string_array"].([]interface{})[0].(map[string]interface{})["asdf"], gs.Equals, 123.0)
c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_string_array"].([]interface{})[1].(map[string]interface{})["jkl;"], gs.Equals, 123.0)
c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_bytes_array"].([]interface{})[0].(map[string]interface{})["asdf"], gs.Equals, 123.0)
c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_bytes_array"].([]interface{})[1].(map[string]interface{})["jkl;"], gs.Equals, 123.0)
})
c.Specify("encodes w/ a different timestamp format", func() {
config.Timestamp = "%Y/%m/%d %H:%M:%S %z"
err := encoder.Init(config)
c.Expect(err, gs.IsNil)
b, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
output := string(b)
lines := strings.Split(output, string(NEWLINE))
decoded := make(map[string]interface{})
err = json.Unmarshal([]byte(lines[1]), &decoded)
c.Assume(err, gs.IsNil)
c.Expect(decoded["@timestamp"], gs.Equals, "2013/07/16 15:49:05 +0000")
})
c.Specify("validates dynamic_fields against fields list", func() {
// "DynamicFields" not listed fails init.
config.DynamicFields = []string{"asdf", "jkl;"}
config.Fields = []string{"Logger", "Hostname"}
err := encoder.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
msg := "\"DynamicFields\" must be in 'fields' list if using 'dynamic_fields'"
c.Expect(err.Error(), gs.Equals, msg)
// "DynamicFields" listed passes init.
config.Fields = []string{"Logger", "Hostname", "DynamicFields"}
err = encoder.Init(config)
c.Expect(err, gs.IsNil)
c.Expect(len(encoder.dynamicFields), gs.Equals, 2)
// "Fields" works as an alias for "DynamicFields".
config.Fields = []string{"Logger", "Hostname", "Fields"}
err = encoder.Init(config)
c.Expect(err, gs.IsNil)
c.Expect(len(encoder.dynamicFields), gs.Equals, 2)
})
c.Specify("honors dynamic fields", func() {
c.Specify("when dynamic_fields is empty", func() {
err := encoder.Init(config)
c.Expect(err, gs.IsNil)
b, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
output := string(b)
lines := strings.Split(output, string(NEWLINE))
decoded := make(map[string]interface{})
err = json.Unmarshal([]byte(lines[1]), &decoded)
c.Assume(err, gs.IsNil)
c.Expect(len(decoded), gs.Equals, 10)
fieldsValInterface := decoded["@fields"]
fieldsVal := fieldsValInterface.(map[string]interface{})
c.Expect(len(fieldsVal), gs.Equals, 13)
})
})
})
c.Specify("ESJsonEncoder", func() {
encoder := new(ESJsonEncoder)
config := encoder.ConfigStruct().(*ESJsonEncoderConfig)
config.RawBytesFields = []string{
"test_raw_field_string",
"test_raw_field_bytes",
"test_raw_field_string_array",
"test_raw_field_bytes_array",
}
c.Specify("Should properly encode a message", func() {
err := encoder.Init(config)
c.Assume(err, gs.IsNil)
b, err := encoder.Encode(pack)
c.Assume(err, gs.IsNil)
output := string(b)
lines := strings.Split(output, string(NEWLINE))
decoded := make(map[string]interface{})
err = json.Unmarshal([]byte(lines[0]), &decoded)
c.Assume(err, gs.IsNil)
sub := decoded["index"].(map[string]interface{})
t := time.Now().UTC()
c.Expect(sub["_index"], gs.Equals, "heka-"+t.Format("2006.01.02"))
c.Expect(sub["_type"], gs.Equals, "message")
decoded = make(map[string]interface{})
err = json.Unmarshal([]byte(lines[1]), &decoded)
c.Assume(err, gs.IsNil)
c.Expect(decoded[`"foo`], gs.Equals, "bar\n")
c.Expect(decoded[`"number`], gs.Equals, 64.0)
c.Expect(decoded["\xEF\xBF\xBD"], gs.Equals, "\xEF\xBF\xBD")
c.Expect(decoded["Uuid"], gs.Equals, "87cf1ac2-e810-4ddf-a02d-a5ce44d13a85")
c.Expect(decoded["Timestamp"], gs.Equals, "2013-07-16T15:49:05")
c.Expect(decoded["Type"], gs.Equals, "TEST")
c.Expect(decoded["Logger"], gs.Equals, "GoSpec")
c.Expect(decoded["Severity"], gs.Equals, 6.0)
c.Expect(decoded["Payload"], gs.Equals, "Test Payload")
c.Expect(decoded["EnvVersion"], gs.Equals, "0.8")
c.Expect(decoded["Pid"], gs.Equals, 14098.0)
c.Expect(decoded["Hostname"], gs.Equals, "hostname")
stringsArray := decoded["stringArray"].([]interface{})
c.Expect(len(stringsArray), gs.Equals, 4)
c.Expect(stringsArray[0], gs.Equals, "asdf")
c.Expect(stringsArray[1], gs.Equals, "jkl;")
c.Expect(stringsArray[2], gs.Equals, "push")
c.Expect(stringsArray[3], gs.Equals, "pull")
bytesArray := decoded["byteArray"].([]interface{})
c.Expect(len(bytesArray), gs.Equals, 2)
c.Expect(bytesArray[0], gs.Equals, base64.StdEncoding.EncodeToString([]byte("asdf")))
c.Expect(bytesArray[1], gs.Equals, base64.StdEncoding.EncodeToString([]byte("jkl;")))
integerArray := decoded["integerArray"].([]interface{})
c.Expect(len(integerArray), gs.Equals, 3)
c.Expect(integerArray[0], gs.Equals, 22.0)
c.Expect(integerArray[1], gs.Equals, 80.0)
c.Expect(integerArray[2], gs.Equals, 3000.0)
doubleArray := decoded["doubleArray"].([]interface{})
c.Expect(len(doubleArray), gs.Equals, 2)
c.Expect(doubleArray[0], gs.Equals, 42.0)
c.Expect(doubleArray[1], gs.Equals, 19101.3)
boolArray := decoded["boolArray"].([]interface{})
c.Expect(len(boolArray), gs.Equals, 5)
c.Expect(boolArray[0], gs.Equals, true)
c.Expect(boolArray[1], gs.Equals, false)
c.Expect(boolArray[2], gs.Equals, false)
c.Expect(boolArray[3], gs.Equals, false)
c.Expect(boolArray[4], gs.Equals, true)
c.Expect(decoded["test_raw_field_string"].(map[string]interface{})["asdf"], gs.Equals, 123.0)
c.Expect(decoded["test_raw_field_bytes"].(map[string]interface{})["asdf"], gs.Equals, 123.0)
c.Expect(decoded["test_raw_field_string_array"].([]interface{})[0].(map[string]interface{})["asdf"], gs.Equals, 123.0)
c.Expect(decoded["test_raw_field_string_array"].([]interface{})[1].(map[string]interface{})["jkl;"], gs.Equals, 123.0)
c.Expect(decoded["test_raw_field_bytes_array"].([]interface{})[0].(map[string]interface{})["asdf"], gs.Equals, 123.0)
c.Expect(decoded["test_raw_field_bytes_array"].([]interface{})[1].(map[string]interface{})["jkl;"], gs.Equals, 123.0)
})
c.Specify("Should use field mappings", func() {
config := encoder.ConfigStruct().(*ESJsonEncoderConfig)
config.FieldMappings = &ESFieldMappings{
Timestamp: "XTimestamp",
Uuid: "XUuid",
Type: "XType",
Logger: "XLogger",
Severity: "XSeverity",
Payload: "XPayload",
EnvVersion: "XEnvVersion",
Pid: "XPid",
Hostname: "XHostname",
}
err := encoder.Init(config)
c.Assume(err, gs.IsNil)
b, err := encoder.Encode(pack)
c.Assume(err, gs.IsNil)
output := string(b)
lines := strings.Split(output, string(NEWLINE))
decoded := make(map[string]interface{})
err = json.Unmarshal([]byte(lines[1]), &decoded)
c.Assume(err, gs.IsNil)
c.Expect(decoded["XTimestamp"], gs.Equals, "2013-07-16T15:49:05")
c.Expect(decoded["XUuid"], gs.Equals, "87cf1ac2-e810-4ddf-a02d-a5ce44d13a85")
c.Expect(decoded["XType"], gs.Equals, "TEST")
c.Expect(decoded["XLogger"], gs.Equals, "GoSpec")
c.Expect(decoded["XSeverity"], gs.Equals, 6.0)
c.Expect(decoded["XPayload"], gs.Equals, "Test Payload")
c.Expect(decoded["XEnvVersion"], gs.Equals, "0.8")
c.Expect(decoded["XPid"], gs.Equals, 14098.0)
c.Expect(decoded["XHostname"], gs.Equals, "hostname")
})
c.Specify("encodes w/ a different timestamp format", func() {
config.Timestamp = "%Y/%m/%d %H:%M:%S %z"
err := encoder.Init(config)
c.Expect(err, gs.IsNil)
b, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
output := string(b)
lines := strings.Split(output, string(NEWLINE))
decoded := make(map[string]interface{})
err = json.Unmarshal([]byte(lines[1]), &decoded)
c.Assume(err, gs.IsNil)
c.Expect(decoded["Timestamp"], gs.Equals, "2013/07/16 15:49:05 +0000")
})
c.Specify("validates field list", func() {
config.Fields = []string{"severity", "Hogger", "Lostname"}
err := encoder.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
msg := "Unsupported value \"Hogger\" in 'fields' list, must be one of"
c.Expect(err.Error()[:len(msg)], gs.Equals, msg)
})
c.Specify("validates dynamic_fields against fields list", func() {
// "DynamicFields" not listed fails init.
config.DynamicFields = []string{"asdf", "jkl;"}
config.Fields = []string{"Logger", "Hostname"}
err := encoder.Init(config)
c.Assume(err, gs.Not(gs.IsNil))
msg := "\"DynamicFields\" must be in 'fields' list if using 'dynamic_fields'"
c.Expect(err.Error(), gs.Equals, msg)
// "DynamicFields" listed passes init.
config.Fields = []string{"Logger", "Hostname", "DynamicFields"}
err = encoder.Init(config)
c.Expect(err, gs.IsNil)
c.Expect(len(encoder.dynamicFields), gs.Equals, 2)
// "Fields" works as an alias for "DynamicFields".
config.Fields = []string{"Logger", "Hostname", "Fields"}
err = encoder.Init(config)
c.Expect(err, gs.IsNil)
c.Expect(len(encoder.dynamicFields), gs.Equals, 2)
})
c.Specify("honors dynamic fields", func() {
c.Specify("when dynamic_fields is empty", func() {
err := encoder.Init(config)
c.Assume(err, gs.IsNil)
b, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
output := string(b)
lines := strings.Split(output, string(NEWLINE))
decoded := make(map[string]interface{})
err = json.Unmarshal([]byte(lines[1]), &decoded)
c.Assume(err, gs.IsNil)
c.Expect(len(decoded), gs.Equals, 22) // 9 base fields and 13 dynamic fields.
})
c.Specify("when dynamic_fields is specified", func() {
config.DynamicFields = []string{"idField", "stringArray"}
err := encoder.Init(config)
c.Assume(err, gs.IsNil)
b, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
output := string(b)
lines := strings.Split(output, string(NEWLINE))
decoded := make(map[string]interface{})
err = json.Unmarshal([]byte(lines[1]), &decoded)
c.Assume(err, gs.IsNil)
c.Expect(len(decoded), gs.Equals, 11) // 9 base fields and 2 dynamic fields.
})
})
})
}
func LoadFromConfigSpec(c gs.Context) {
origGlobals := Globals
origAvailablePlugins := make(map[string]func() interface{})
for k, v := range AvailablePlugins {
origAvailablePlugins[k] = v
}
pipeConfig := NewPipelineConfig(nil)
defer func() {
Globals = origGlobals
AvailablePlugins = origAvailablePlugins
}()
c.Assume(pipeConfig, gs.Not(gs.IsNil))
c.Specify("Config file loading", func() {
c.Specify("works w/ good config file", func() {
err := pipeConfig.LoadFromConfigFile("./testsupport/config_test.toml")
c.Assume(err, gs.IsNil)
// We use a set of Expect's rather than c.Specify because the
// pipeConfig can't be re-loaded per child as gospec will do
// since each one needs to bind to the same address
// and the inputs section loads properly with a custom name
udp, ok := pipeConfig.InputRunners["UdpInput"]
c.Expect(ok, gs.Equals, true)
// and the decoders sections load
_, ok = pipeConfig.DecoderWrappers["JsonDecoder"]
c.Expect(ok, gs.Equals, false)
_, ok = pipeConfig.DecoderWrappers["ProtobufDecoder"]
c.Expect(ok, gs.Equals, true)
// and the outputs section loads
_, ok = pipeConfig.OutputRunners["LogOutput"]
c.Expect(ok, gs.Equals, true)
// and the filters sections loads
_, ok = pipeConfig.FilterRunners["sample"]
c.Expect(ok, gs.Equals, true)
// Shut down UdpInput to free up the port for future tests.
udp.Input().Stop()
})
c.Specify("works w/ decoder defaults", func() {
err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_defaults.toml")
c.Assume(err, gs.Not(gs.IsNil))
// Only the ProtobufDecoder is loaded
c.Expect(len(pipeConfig.DecoderWrappers), gs.Equals, 1)
})
c.Specify("works w/ MultiDecoder", func() {
err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_multidecoder.toml")
c.Assume(err, gs.IsNil)
hasSyncDecoder := false
// ProtobufDecoder will always be loaded
c.Assume(len(pipeConfig.DecoderWrappers), gs.Equals, 2)
// Check that the MultiDecoder actually loaded
for k, _ := range pipeConfig.DecoderWrappers {
if k == "syncdecoder" {
hasSyncDecoder = true
break
}
}
c.Assume(hasSyncDecoder, gs.IsTrue)
})
c.Specify("explodes w/ bad config file", func() {
err := pipeConfig.LoadFromConfigFile("./testsupport/config_bad_test.toml")
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), ts.StringContains, "2 errors loading plugins")
c.Expect(pipeConfig.LogMsgs, gs.ContainsAny, gs.Values("No such plugin: CounterOutput"))
})
c.Specify("handles missing config file correctly", func() {
err := pipeConfig.LoadFromConfigFile("no_such_file.toml")
c.Assume(err, gs.Not(gs.IsNil))
if runtime.GOOS == "windows" {
c.Expect(err.Error(), ts.StringContains, "open no_such_file.toml: The system cannot find the file specified.")
} else {
c.Expect(err.Error(), ts.StringContains, "open no_such_file.toml: no such file or directory")
}
})
c.Specify("errors correctly w/ bad outputs config", func() {
err := pipeConfig.LoadFromConfigFile("./testsupport/config_bad_outputs.toml")
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), ts.StringContains, "1 errors loading plugins")
msg := pipeConfig.LogMsgs[0]
c.Expect(msg, ts.StringContains, "No such plugin")
})
c.Specify("for a DefaultsTestOutput", func() {
RegisterPlugin("DefaultsTestOutput", func() interface{} {
return new(DefaultsTestOutput)
})
err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_defaults2.toml")
c.Expect(err, gs.IsNil)
runner, ok := pipeConfig.OutputRunners["DefaultsTestOutput"]
c.Expect(ok, gs.IsTrue)
ticker := runner.Ticker()
c.Expect(ticker, gs.Not(gs.IsNil))
matcher := runner.MatchRunner().MatcherSpecification().String()
c.Expect(matcher, gs.Equals, messageMatchStr)
})
c.Specify("can render JSON reports as pipe delimited data", func() {
RegisterPlugin("DefaultsTestOutput", func() interface{} {
return new(DefaultsTestOutput)
})
err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_defaults2.toml")
c.Expect(err, gs.IsNil)
data := `{"globals":[{"Name":"inputRecycleChan","InChanCapacity":{"value":"100", "representation":"count"},"InChanLength":{"value":"99", "representation":"count"}},{"Name":"injectRecycleChan","InChanCapacity":{"value":"100", "representation":"count"},"InChanLength":{"value":"98", "representation":"count"}},{"Name":"Router","InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"ProcessMessageCount":{"value":"26", "representation":"count"}}], "inputs": [{"Name": "TcpInput"}], "decoders": [{"Name":"ProtobufDecoder","InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"}}], "filters": [{"Name":"OpsSandboxManager","RunningFilters":{"value":"0", "representation":"count"},"ProcessMessageCount":{"value":"0", "representation":"count"},"InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"MatchChanCapacity":{"value":"50", "representation":"count"},"MatchChanLength":{"value":"0", "representation":"count"},"MatchAvgDuration":{"value":"0", "representation":"ns"}},{"Name":"hekabench_counter","Memory":{"value":"20644", "representation":"B"},"MaxMemory":{"value":"20644", "representation":"B"},"MaxInstructions":{"value":"18", "representation":"count"},"MaxOutput":{"value":"0", "representation":"B"},"ProcessMessageCount":{"value":"0", "representation":"count"},"InjectMessageCount":{"value":"0", "representation":"count"},"ProcessMessageAvgDuration":{"value":"0", "representation":"ns"},"TimerEventAvgDuration":{"value":"78532", "representation":"ns"},"InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"MatchChanCapacity":{"value":"50", "representation":"count"},"MatchChanLength":{"value":"0", "representation":"count"},"MatchAvgDuration":{"value":"445", "representation":"ns"}}], "outputs": [{"Name":"LogOutput","InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"MatchChanCapacity":{"value":"50", "representation":"count"},"MatchChanLength":{"value":"0", "representation":"count"},"MatchAvgDuration":{"value":"406", "representation":"ns"}},{"Name":"DashboardOutput","InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"MatchChanCapacity":{"value":"50", "representation":"count"},"MatchChanLength":{"value":"0", "representation":"count"},"MatchAvgDuration":{"value":"336", "representation":"ns"}}]} `
report := pipeConfig.FormatTextReport("heka.all-report", data)
expected := `========[heka.all-report]========
====Globals====
inputRecycleChan:
InChanCapacity: 100
InChanLength: 99
injectRecycleChan:
InChanCapacity: 100
InChanLength: 98
Router:
InChanCapacity: 50
InChanLength: 0
ProcessMessageCount: 26
====Inputs====
TcpInput:
====Decoders====
ProtobufDecoder:
InChanCapacity: 50
InChanLength: 0
====Filters====
OpsSandboxManager:
InChanCapacity: 50
InChanLength: 0
MatchChanCapacity: 50
MatchChanLength: 0
MatchAvgDuration: 0
ProcessMessageCount: 0
hekabench_counter:
InChanCapacity: 50
InChanLength: 0
MatchChanCapacity: 50
MatchChanLength: 0
MatchAvgDuration: 445
ProcessMessageCount: 0
InjectMessageCount: 0
Memory: 20644
MaxMemory: 20644
MaxInstructions: 18
MaxOutput: 0
ProcessMessageAvgDuration: 0
TimerEventAvgDuration: 78532
====Outputs====
LogOutput:
InChanCapacity: 50
InChanLength: 0
MatchChanCapacity: 50
MatchChanLength: 0
MatchAvgDuration: 406
DashboardOutput:
InChanCapacity: 50
InChanLength: 0
MatchChanCapacity: 50
MatchChanLength: 0
MatchAvgDuration: 336
========
`
c.Expect(report, gs.Equals, expected)
})
c.Specify("works w/ bad param config file", func() {
err := pipeConfig.LoadFromConfigFile("./testsupport/config_bad_params.toml")
c.Assume(err, gs.Not(gs.IsNil))
})
c.Specify("works w/ common parameters that are not part of the struct", func() {
err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_common.toml")
c.Assume(err, gs.IsNil)
})
})
c.Specify("Config directory helpers", func() {
Globals().BaseDir = "/base/dir"
Globals().ShareDir = "/share/dir"
c.Specify("PrependBaseDir", func() {
c.Specify("prepends for relative paths", func() {
dir := filepath.FromSlash("relative/path")
result := PrependBaseDir(dir)
c.Expect(result, gs.Equals, filepath.FromSlash("/base/dir/relative/path"))
})
c.Specify("doesn't prepend for absolute paths", func() {
dir := filepath.FromSlash("/absolute/path")
result := PrependBaseDir(dir)
c.Expect(result, gs.Equals, dir)
})
})
c.Specify("PrependShareDir", func() {
c.Specify("prepends for relative paths", func() {
dir := filepath.FromSlash("relative/path")
result := PrependShareDir(dir)
c.Expect(result, gs.Equals, filepath.FromSlash("/share/dir/relative/path"))
})
c.Specify("doesn't prepend for absolute paths", func() {
dir := filepath.FromSlash("/absolute/path")
result := PrependShareDir(dir)
c.Expect(result, gs.Equals, dir)
})
})
})
c.Specify("PluginHelper", func() {
c.Specify("starts and stops DecoderRunners appropriately", func() {
err := pipeConfig.LoadFromConfigFile("./testsupport/config_test.toml")
c.Assume(err, gs.IsNil)
// Start two DecoderRunners.
dr1, ok := pipeConfig.DecoderRunner("ProtobufDecoder", "ProtobufDecoder_1")
c.Expect(ok, gs.IsTrue)
dr2, ok := pipeConfig.DecoderRunner("ProtobufDecoder", "ProtobufDecoder_2")
c.Expect(ok, gs.IsTrue)
// Stop the second one.
ok = pipeConfig.StopDecoderRunner(dr2)
c.Expect(ok, gs.IsTrue)
// Verify that it's stopped, i.e. InChan is closed.
_, ok = <-dr2.InChan()
c.Expect(ok, gs.IsFalse)
// Verify that dr1 is *not* stopped, i.e. InChan is still open.
rChan := make(chan *PipelinePack, 1)
pack := NewPipelinePack(rChan)
dr1.InChan() <- pack // <-- Failure case means this will panic.
// Try to stop dr2 again. Shouldn't fail, but ok should be false.
ok = pipeConfig.StopDecoderRunner(dr2)
c.Expect(ok, gs.IsFalse)
// Clean up our UdpInput.
udp, _ := pipeConfig.InputRunners["UdpInput"]
udp.Input().Stop()
})
})
}
func LoadFromConfigSpec(c gs.Context) {
c.Specify("Config file loading", func() {
origGlobals := Globals
origDecodersByEncoding := DecodersByEncoding
pipeConfig := NewPipelineConfig(nil)
defer func() {
Globals = origGlobals
DecodersByEncoding = origDecodersByEncoding
}()
c.Assume(pipeConfig, gs.Not(gs.IsNil))
c.Specify("works w/ good config file", func() {
err := pipeConfig.LoadFromConfigFile("../testsupport/config_test.toml")
c.Assume(err, gs.IsNil)
// We use a set of Expect's rather than c.Specify because the
// pipeConfig can't be re-loaded per child as gospec will do
// since each one needs to bind to the same address
// and the decoders are loaded for the right encoding headers
dSet := pipeConfig.DecoderSets[0]
dRunner, ok := dSet.ByEncoding(message.Header_JSON)
c.Expect(dRunner, gs.Not(gs.IsNil))
c.Expect(ok, gs.IsTrue)
c.Expect(dRunner.Name(), gs.Equals, "JsonDecoder")
dRunner, ok = dSet.ByEncoding(message.Header_PROTOCOL_BUFFER)
c.Expect(dRunner, gs.Not(gs.IsNil))
c.Expect(ok, gs.IsTrue)
c.Expect(dRunner.Name(), gs.Equals, "ProtobufDecoder")
// decoders channel is full
c.Expect(len(pipeConfig.decodersChan), gs.Equals, Globals().DecoderPoolSize)
// and the inputs section loads properly with a custom name
_, ok = pipeConfig.InputRunners["UdpInput"]
c.Expect(ok, gs.Equals, true)
// and the decoders sections load
_, ok = pipeConfig.DecoderWrappers["JsonDecoder"]
c.Expect(ok, gs.Equals, true)
_, ok = pipeConfig.DecoderWrappers["ProtobufDecoder"]
c.Expect(ok, gs.Equals, true)
// and the outputs section loads
_, ok = pipeConfig.OutputRunners["LogOutput"]
c.Expect(ok, gs.Equals, true)
// and the filters sections loads
_, ok = pipeConfig.FilterRunners["sample"]
c.Expect(ok, gs.Equals, true)
})
c.Specify("works w/ decoder defaults", func() {
err := pipeConfig.LoadFromConfigFile("../testsupport/config_test_defaults.toml")
c.Assume(err, gs.Not(gs.IsNil))
// Decoders are loaded
c.Expect(len(pipeConfig.DecoderWrappers), gs.Equals, 2)
c.Expect(DecodersByEncoding[message.Header_JSON], gs.Equals, "JsonDecoder")
c.Expect(DecodersByEncoding[message.Header_PROTOCOL_BUFFER], gs.Equals,
"ProtobufDecoder")
})
c.Specify("explodes w/ bad config file", func() {
err := pipeConfig.LoadFromConfigFile("../testsupport/config_bad_test.toml")
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), ts.StringContains, "2 errors loading plugins")
c.Expect(pipeConfig.logMsgs, gs.ContainsAny, gs.Values("No such plugin: CounterOutput"))
})
c.Specify("handles missing config file correctly", func() {
err := pipeConfig.LoadFromConfigFile("no_such_file.toml")
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), ts.StringContains, "open no_such_file.toml: no such file or directory")
})
c.Specify("errors correctly w/ bad outputs config", func() {
err := pipeConfig.LoadFromConfigFile("../testsupport/config_bad_outputs.toml")
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), ts.StringContains, "1 errors loading plugins")
msg := pipeConfig.logMsgs[0]
c.Expect(msg, ts.StringContains, "No such plugin")
})
c.Specify("captures plugin Init() panics", func() {
RegisterPlugin("PanicOutput", func() interface{} {
return new(PanicOutput)
})
err := pipeConfig.LoadFromConfigFile("../testsupport/config_panic.toml")
c.Expect(err, gs.Not(gs.IsNil))
})
})
}
func DecodersSpec(c gospec.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
msg := getTestMessage()
config := NewPipelineConfig(nil)
c.Specify("A ProtobufDecoder", func() {
encoded, err := proto.Marshal(msg)
c.Assume(err, gs.IsNil)
pack := NewPipelinePack(config.inputRecycleChan)
decoder := new(ProtobufDecoder)
c.Specify("decodes a protobuf message", func() {
pack.MsgBytes = encoded
err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
c.Expect(pack.Message, gs.Equals, msg)
v, ok := pack.Message.GetFieldValue("foo")
c.Expect(ok, gs.IsTrue)
c.Expect(v, gs.Equals, "bar")
})
c.Specify("returns an error for bunk encoding", func() {
bunk := append([]byte{0, 0, 0}, encoded...)
pack.MsgBytes = bunk
err := decoder.Decode(pack)
c.Expect(err, gs.Not(gs.IsNil))
})
})
c.Specify("A MultiDecoder", func() {
decoder := new(MultiDecoder)
conf := decoder.ConfigStruct().(*MultiDecoderConfig)
supply := make(chan *PipelinePack, 1)
pack := NewPipelinePack(supply)
conf.Name = "MyMultiDecoder"
conf.Subs = make(map[string]interface{}, 0)
conf.Subs["StartsWithM"] = make(map[string]interface{}, 0)
withM := conf.Subs["StartsWithM"].(map[string]interface{})
withM["type"] = "PayloadRegexDecoder"
withM["match_regex"] = "^(?P<TheData>m.*)"
withMFields := make(map[string]interface{}, 0)
withMFields["StartsWithM"] = "%TheData%"
withM["message_fields"] = withMFields
conf.Order = []string{"StartsWithM"}
errMsg := "Unable to decode message with any contained decoder."
dRunner := NewMockDecoderRunner(ctrl)
// An error will be spit out b/c there's no real *dRunner in there;
// doesn't impact the tests.
dRunner.EXPECT().LogError(gomock.Any())
c.Specify("decodes simple messages", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
regex_data := "matching text"
pack.Message.SetPayload(regex_data)
err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetType(), gs.Equals, "heka.MyMultiDecoder")
value, ok := pack.Message.GetFieldValue("StartsWithM")
c.Assume(ok, gs.IsTrue)
c.Expect(value, gs.Equals, regex_data)
})
c.Specify("returns an error if all decoders fail", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
regex_data := "non-matching text"
pack.Message.SetPayload(regex_data)
err = decoder.Decode(pack)
c.Expect(err.Error(), gs.Equals, errMsg)
})
c.Specify("logs subdecoder failures when configured to do so", func() {
conf.LogSubErrors = true
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
regex_data := "non-matching text"
pack.Message.SetPayload(regex_data)
// Expect that we log an error for undecoded message.
dRunner.EXPECT().LogError(fmt.Errorf("Subdecoder 'StartsWithM' decode error: No match"))
err = decoder.Decode(pack)
c.Expect(err.Error(), gs.Equals, errMsg)
})
c.Specify("sets subdecoder runner correctly", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
// Call LogError to appease the angry gomock gods.
dRunner.LogError(errors.New("foo"))
// Now create a real *dRunner, pass it in, make sure a wrapper
// gets handed to the subdecoder.
dr := NewDecoderRunner(decoder.Name, decoder, new(PluginGlobals))
decoder.SetDecoderRunner(dr)
sub := decoder.Decoders["StartsWithM"]
subRunner := sub.(*PayloadRegexDecoder).dRunner
subRunner, ok := subRunner.(*mDRunner)
c.Expect(ok, gs.IsTrue)
c.Expect(subRunner.Name(), gs.Equals,
fmt.Sprintf("%s-StartsWithM", decoder.Name))
c.Expect(subRunner.Decoder(), gs.Equals, sub)
})
c.Specify("with multiple registered decoders", func() {
conf.Subs["StartsWithS"] = make(map[string]interface{}, 0)
withS := conf.Subs["StartsWithS"].(map[string]interface{})
withS["type"] = "PayloadRegexDecoder"
withS["match_regex"] = "^(?P<TheData>s.*)"
withSFields := make(map[string]interface{}, 0)
withSFields["StartsWithS"] = "%TheData%"
withS["message_fields"] = withSFields
conf.Subs["StartsWithM2"] = make(map[string]interface{}, 0)
withM2 := conf.Subs["StartsWithM2"].(map[string]interface{})
withM2["type"] = "PayloadRegexDecoder"
withM2["match_regex"] = "^(?P<TheData>m.*)"
withM2Fields := make(map[string]interface{}, 0)
withM2Fields["StartsWithM2"] = "%TheData%"
withM2["message_fields"] = withM2Fields
conf.Order = append(conf.Order, "StartsWithS", "StartsWithM2")
var ok bool
// Two more subdecoders means two more LogError calls.
dRunner.EXPECT().LogError(gomock.Any()).Times(2)
c.Specify("defaults to `first-wins` cascading", func() {
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("on a first match condition", func() {
regexData := "match first"
pack.Message.SetPayload(regexData)
err = decoder.Decode(pack)
c.Expect(err, gs.IsNil)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsTrue)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
c.Specify("and a second match condition", func() {
regexData := "second match"
pack.Message.SetPayload(regexData)
err = decoder.Decode(pack)
c.Expect(err, gs.IsNil)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsTrue)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
c.Specify("returning an error if they all fail", func() {
regexData := "won't match"
pack.Message.SetPayload(regexData)
err = decoder.Decode(pack)
c.Expect(err.Error(), gs.Equals, errMsg)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
})
c.Specify("and using `all` cascading", func() {
conf.CascadeStrategy = "all"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
decoder.SetDecoderRunner(dRunner)
c.Specify("matches multiples when appropriate", func() {
regexData := "matches twice"
pack.Message.SetPayload(regexData)
err = decoder.Decode(pack)
c.Expect(err, gs.IsNil)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsTrue)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsTrue)
})
c.Specify("matches singles when appropriate", func() {
regexData := "second match"
pack.Message.SetPayload(regexData)
err = decoder.Decode(pack)
c.Expect(err, gs.IsNil)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsTrue)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
c.Specify("returns an error if they all fail", func() {
regexData := "won't match"
pack.Message.SetPayload(regexData)
err = decoder.Decode(pack)
c.Expect(err.Error(), gs.Equals, errMsg)
_, ok = pack.Message.GetFieldValue("StartsWithM")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithS")
c.Expect(ok, gs.IsFalse)
_, ok = pack.Message.GetFieldValue("StartsWithM2")
c.Expect(ok, gs.IsFalse)
})
})
})
})
c.Specify("A PayloadJsonDecoder", func() {
decoder := new(PayloadJsonDecoder)
conf := decoder.ConfigStruct().(*PayloadJsonDecoderConfig)
supply := make(chan *PipelinePack, 1)
pack := NewPipelinePack(supply)
c.Specify("decodes simple messages", func() {
json_data := `{"statsd": {"count": 1, "name": "some.counter"}, "pid": 532, "timestamp": "2013-08-13T10:32:00.000Z"}`
conf.JsonMap = map[string]string{"Count": "$.statsd.count",
"Name": "$.statsd.name",
"Pid": "$.pid",
"Timestamp": "$.timestamp",
}
conf.MessageFields = MessageTemplate{
"Pid": "%Pid%",
"StatCount": "%Count%",
"StatName": "%Name%",
"Timestamp": "%Timestamp%",
}
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(json_data)
err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetPid(), gs.Equals, int32(532))
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1376389920000000000))
var ok bool
var name, count interface{}
count, ok = pack.Message.GetFieldValue("StatCount")
c.Expect(ok, gs.Equals, true)
c.Expect(count, gs.Equals, "1")
name, ok = pack.Message.GetFieldValue("StatName")
c.Expect(ok, gs.Equals, true)
c.Expect(name, gs.Equals, "some.counter")
})
})
c.Specify("A PayloadXmlDecoder", func() {
decoder := new(PayloadXmlDecoder)
conf := decoder.ConfigStruct().(*PayloadXmlDecoderConfig)
supply := make(chan *PipelinePack, 1)
pack := NewPipelinePack(supply)
c.Specify("decodes simple messages", func() {
xml_data := `<library>
<!-- Great book. -->
<book id="b0836217462" available="true">
<isbn>0836217462</isbn>
<title lang="en">Being a Dog Is a Full-Time Job</title>
<quote>I'd dog paddle the deepest ocean.</quote>
<author id="CMS">
<?echo "go rocks"?>
<name>Charles M Schulz</name>
<born>1922-11-26</born>
<dead>2000-02-12</dead>
</author>
<character id="PP">
<name>Peppermint Patty</name>
<born>1966-08-22</born>
<qualificati>bold, brash and tomboyish</qualificati>
</character>
<character id="Snoopy">
<name>Snoopy</name>
<born>1950-10-04</born>
<qualificati>extroverted beagle</qualificati>
</character>
</book>
</library>`
conf.XPathMapConfig = map[string]string{"Isbn": "library/*/isbn",
"Name": "/library/book/character[born='1950-10-04']/name",
"Patty": "/library/book//node()[@id='PP']/name",
"Title": "//book[author/@id='CMS']/title",
"Comment": "/library/book/preceding::comment()",
}
conf.MessageFields = MessageTemplate{
"Isbn": "%Isbn%",
"Name": "%Name%",
"Patty": "%Patty%",
"Title": "%Title%",
"Comment": "%Comment%",
}
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(xml_data)
err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
var isbn, name, patty, title, comment interface{}
var ok bool
isbn, ok = pack.Message.GetFieldValue("Isbn")
c.Expect(ok, gs.Equals, true)
name, ok = pack.Message.GetFieldValue("Name")
c.Expect(ok, gs.Equals, true)
patty, ok = pack.Message.GetFieldValue("Patty")
c.Expect(ok, gs.Equals, true)
title, ok = pack.Message.GetFieldValue("Title")
c.Expect(ok, gs.Equals, true)
comment, ok = pack.Message.GetFieldValue("Comment")
c.Expect(ok, gs.Equals, true)
c.Expect(isbn, gs.Equals, "0836217462")
c.Expect(name, gs.Equals, "Snoopy")
c.Expect(patty, gs.Equals, "Peppermint Patty")
c.Expect(title, gs.Equals, "Being a Dog Is a Full-Time Job")
c.Expect(comment, gs.Equals, " Great book. ")
})
})
c.Specify("A PayloadRegexDecoder", func() {
decoder := new(PayloadRegexDecoder)
conf := decoder.ConfigStruct().(*PayloadRegexDecoderConfig)
supply := make(chan *PipelinePack, 1)
pack := NewPipelinePack(supply)
conf.TimestampLayout = "02/Jan/2006:15:04:05 -0700"
c.Specify("non capture regex", func() {
conf.MatchRegex = `\d+`
err := decoder.Init(conf)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "PayloadRegexDecoder regex must contain capture groups")
})
c.Specify("invalid regex", func() {
conf.MatchRegex = `\mtest`
err := decoder.Init(conf)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "PayloadRegexDecoder: error parsing regexp: invalid escape sequence: `\\m`")
})
c.Specify("reading an apache timestamp", func() {
conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]`
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload("[18/Apr/2013:14:00:28 -0700]")
err = decoder.Decode(pack)
c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1366318828000000000))
pack.Zero()
})
c.Specify("uses kitchen timestamp", func() {
conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]`
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload("[5:16PM]")
now := time.Now()
cur_date := time.Date(now.Year(), now.Month(), now.Day(), 17, 16, 0, 0,
time.UTC)
err = decoder.Decode(pack)
c.Expect(pack.Message.GetTimestamp(), gs.Equals, cur_date.UnixNano())
pack.Zero()
})
c.Specify("adjusts timestamps as specified", func() {
conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]`
conf.TimestampLayout = "02/Jan/2006:15:04:05"
conf.TimestampLocation = "America/Los_Angeles"
timeStr := "18/Apr/2013:14:00:28"
loc, err := time.LoadLocation(conf.TimestampLocation)
c.Assume(err, gs.IsNil)
expectedLocal, err := time.ParseInLocation(conf.TimestampLayout, timeStr, loc)
c.Assume(err, gs.IsNil)
err = decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload("[" + timeStr + "]")
err = decoder.Decode(pack)
c.Expect(pack.Message.GetTimestamp(), gs.Equals, expectedLocal.UnixNano())
pack.Zero()
})
c.Specify("apply representation metadata to a captured field", func() {
value := "0.23"
payload := "header"
conf.MatchRegex = `(?P<ResponseTime>\d+\.\d+)`
conf.MessageFields = MessageTemplate{
"ResponseTime|s": "%ResponseTime%",
"Payload|s": "%ResponseTime%",
"Payload": payload,
}
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(value)
err = decoder.Decode(pack)
f := pack.Message.FindFirstField("ResponseTime")
c.Expect(f, gs.Not(gs.IsNil))
c.Expect(f.GetValue(), gs.Equals, value)
c.Expect(f.GetRepresentation(), gs.Equals, "s")
f = pack.Message.FindFirstField("Payload")
c.Expect(f, gs.Not(gs.IsNil))
c.Expect(f.GetValue(), gs.Equals, value)
c.Expect(f.GetRepresentation(), gs.Equals, "s")
c.Expect(pack.Message.GetPayload(), gs.Equals, payload)
pack.Zero()
})
c.Specify("reading test-zeus.log", func() {
conf.MatchRegex = `(?P<Ip>([0-9]{1,3}\.){3}[0-9]{1,3}) (?P<Hostname>(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])) (?P<User>\w+) \[(?P<Timestamp>[^\]]+)\] \"(?P<Verb>[A-X]+) (?P<Request>\/\S*) HTTP\/(?P<Httpversion>\d\.\d)\" (?P<Response>\d{3}) (?P<Bytes>\d+)`
conf.MessageFields = MessageTemplate{
"hostname": "%Hostname%",
"ip": "%Ip%",
"response": "%Response%",
}
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
filePath := "../testsupport/test-zeus.log"
fileBytes, err := ioutil.ReadFile(filePath)
c.Assume(err, gs.IsNil)
fileStr := string(fileBytes)
lines := strings.Split(fileStr, "\n")
containsFieldValue := func(str, fieldName string, msg *message.Message) bool {
raw, ok := msg.GetFieldValue(fieldName)
if !ok {
return false
}
value := raw.(string)
return strings.Contains(str, value)
}
c.Specify("extracts capture data and puts it in the message fields", func() {
var misses int
for _, line := range lines {
if strings.TrimSpace(line) == "" {
continue
}
pack.Message.SetPayload(line)
err = decoder.Decode(pack)
if err != nil {
misses++
continue
}
c.Expect(containsFieldValue(line, "hostname", pack.Message), gs.IsTrue)
c.Expect(containsFieldValue(line, "ip", pack.Message), gs.IsTrue)
c.Expect(containsFieldValue(line, "response", pack.Message), gs.IsTrue)
pack.Zero()
}
c.Expect(misses, gs.Equals, 3)
})
})
c.Specify("reading test-severity.log", func() {
conf.MatchRegex = `severity: (?P<Severity>[a-zA-Z]+)`
conf.SeverityMap = map[string]int32{
"emergency": 0,
"alert": 1,
"critical": 2,
"error": 3,
"warning": 4,
"notice": 5,
"info": 6,
"debug": 7,
}
reverseMap := make(map[int32]string)
for str, i := range conf.SeverityMap {
reverseMap[i] = str
}
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
filePath := "../testsupport/test-severity.log"
fileBytes, err := ioutil.ReadFile(filePath)
c.Assume(err, gs.IsNil)
fileStr := string(fileBytes)
lines := strings.Split(fileStr, "\n")
c.Specify("sets message severity based on SeverityMap", func() {
err := errors.New("Don't recognize severity: 'BOGUS'")
dRunner.EXPECT().LogError(err)
for _, line := range lines {
if strings.TrimSpace(line) == "" {
continue
}
pack.Message.SetPayload(line)
err = decoder.Decode(pack)
if err != nil {
fmt.Println(line)
}
c.Expect(err, gs.IsNil)
if strings.Contains(line, "BOGUS") {
continue
}
strVal := reverseMap[pack.Message.GetSeverity()]
c.Expect(strings.Contains(line, strVal), gs.IsTrue)
}
})
})
})
c.Specify("A SandboxDecoder", func() {
decoder := new(SandboxDecoder)
conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
conf.ScriptFilename = "../sandbox/lua/testsupport/decoder.lua"
conf.ScriptType = "lua"
supply := make(chan *PipelinePack, 1)
pack := NewPipelinePack(supply)
c.Specify("decodes simple messages", func() {
data := "1376389920 debug id=2321 url=example.com item=1"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
err = decoder.Decode(pack)
c.Assume(err, gs.IsNil)
c.Expect(pack.Message.GetTimestamp(),
gs.Equals,
int64(1376389920000000000))
c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))
var ok bool
var value interface{}
value, ok = pack.Message.GetFieldValue("id")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "2321")
value, ok = pack.Message.GetFieldValue("url")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "example.com")
value, ok = pack.Message.GetFieldValue("item")
c.Expect(ok, gs.Equals, true)
c.Expect(value, gs.Equals, "1")
})
c.Specify("decodes an invalid messages", func() {
data := "1376389920 bogus id=2321 url=example.com item=1"
err := decoder.Init(conf)
c.Assume(err, gs.IsNil)
dRunner := NewMockDecoderRunner(ctrl)
decoder.SetDecoderRunner(dRunner)
pack.Message.SetPayload(data)
err = decoder.Decode(pack)
c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
})
})
c.Specify("A StatsToFieldsDecoder", func() {
decoder := new(StatsToFieldsDecoder)
router := NewMessageRouter()
router.inChan = make(chan *PipelinePack, 5)
dRunner := NewMockDecoderRunner(ctrl)
decoder.runner = dRunner
dRunner.EXPECT().Router().Return(router)
pack := NewPipelinePack(config.inputRecycleChan)
mergeStats := func(stats [][]string) string {
lines := make([]string, len(stats))
for i, line := range stats {
lines[i] = strings.Join(line, " ")
}
return strings.Join(lines, "\n")
}
c.Specify("correctly converts stats to fields", func() {
stats := [][]string{
{"stat.one", "1", "1380047333"},
{"stat.two", "2", "1380047333"},
{"stat.three", "3", "1380047333"},
{"stat.four", "4", "1380047333"},
{"stat.five", "5", "1380047333"},
}
pack.Message.SetPayload(mergeStats(stats))
err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
for i, stats := range stats {
value, ok := pack.Message.GetFieldValue(stats[0])
c.Expect(ok, gs.IsTrue)
expected := float64(i + 1)
c.Expect(value.(float64), gs.Equals, expected)
}
value, ok := pack.Message.GetFieldValue("timestamp")
c.Expect(ok, gs.IsTrue)
expected, err := strconv.ParseInt(stats[0][2], 0, 32)
c.Assume(err, gs.IsNil)
c.Expect(value.(int64), gs.Equals, expected)
})
c.Specify("generates multiple messages for multiple timestamps", func() {
stats := [][]string{
{"stat.one", "1", "1380047333"},
{"stat.two", "2", "1380047333"},
{"stat.three", "3", "1380047331"},
{"stat.four", "4", "1380047333"},
{"stat.five", "5", "1380047332"},
}
// Prime the pack supply w/ two new packs.
dRunner.EXPECT().NewPack().Return(NewPipelinePack(nil))
dRunner.EXPECT().NewPack().Return(NewPipelinePack(nil))
// Decode and check the main pack.
pack.Message.SetPayload(mergeStats(stats))
err := decoder.Decode(pack)
c.Expect(err, gs.IsNil)
value, ok := pack.Message.GetFieldValue("timestamp")
c.Expect(ok, gs.IsTrue)
expected, err := strconv.ParseInt(stats[0][2], 0, 32)
c.Assume(err, gs.IsNil)
c.Expect(value.(int64), gs.Equals, expected)
// Check the first extra.
pack = <-router.inChan
value, ok = pack.Message.GetFieldValue("timestamp")
c.Expect(ok, gs.IsTrue)
expected, err = strconv.ParseInt(stats[2][2], 0, 32)
c.Assume(err, gs.IsNil)
c.Expect(value.(int64), gs.Equals, expected)
// Check the second extra.
pack = <-router.inChan
value, ok = pack.Message.GetFieldValue("timestamp")
c.Expect(ok, gs.IsTrue)
expected, err = strconv.ParseInt(stats[4][2], 0, 32)
c.Assume(err, gs.IsNil)
c.Expect(value.(int64), gs.Equals, expected)
})
c.Specify("fails w/ invalid timestamp", func() {
stats := [][]string{
{"stat.one", "1", "1380047333"},
{"stat.two", "2", "1380047333"},
{"stat.three", "3", "1380047333c"},
{"stat.four", "4", "1380047333"},
{"stat.five", "5", "1380047332"},
}
pack.Message.SetPayload(mergeStats(stats))
err := decoder.Decode(pack)
c.Expect(err, gs.Not(gs.IsNil))
expected := fmt.Sprintf("invalid timestamp: '%s'",
strings.Join(stats[2], " "))
c.Expect(err.Error(), gs.Equals, expected)
})
c.Specify("fails w/ invalid value", func() {
stats := [][]string{
{"stat.one", "1", "1380047333"},
{"stat.two", "2", "1380047333"},
{"stat.three", "3", "1380047333"},
{"stat.four", "4d", "1380047333"},
{"stat.five", "5", "1380047332"},
}
pack.Message.SetPayload(mergeStats(stats))
err := decoder.Decode(pack)
c.Expect(err, gs.Not(gs.IsNil))
expected := fmt.Sprintf("invalid value: '%s'",
strings.Join(stats[3], " "))
c.Expect(err.Error(), gs.Equals, expected)
})
})
}
func MessageEqualsSpec(c gospec.Context) {
msg0 := getTestMessage()
c.Specify("Messages are equal", func() {
msg1 := CopyMessage(msg0)
c.Expect(msg0, gs.Equals, msg1)
})
c.Specify("Messages w/ diff severity", func() {
msg1 := CopyMessage(msg0)
*msg1.Severity--
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ diff uuid", func() {
msg1 := CopyMessage(msg0)
u := uuid.NewRandom()
copy(msg1.Uuid, u)
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ diff payload", func() {
msg1 := CopyMessage(msg0)
*msg1.Payload = "Something completely different"
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ diff number of fields", func() {
msg1 := CopyMessage(msg0)
f, _ := NewField("sna", "foo", "")
msg1.AddField(f)
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ diff number of field values in a key", func() {
msg1 := CopyMessage(msg0)
f := msg1.FindFirstField("foo")
f.AddValue("foo1")
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ diff value in a field", func() {
msg1 := CopyMessage(msg0)
f := msg1.FindFirstField("foo")
f.ValueString[0] = "bah"
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ diff field key", func() {
msg1 := CopyMessage(msg0)
f := msg1.FindFirstField("foo")
*f.Name = "widget"
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
c.Specify("Messages w/ recurring keys", func() {
msg0 = &Message{}
f, _ := NewField("foo", "bar", "")
f1, _ := NewField("foo", "bar1", "")
msg0.AddField(f)
msg0.AddField(f1)
msg1 := CopyMessage(msg0)
c.Expect(msg0, gs.Equals, msg1)
foos := msg0.FindAllFields("foo")
foos[1].ValueString[0] = "bar2"
c.Expect(msg0, gs.Not(gs.Equals), msg1)
})
}
func FilehandlingSpec(c gs.Context) {
here, err := os.Getwd()
c.Assume(err, gs.IsNil)
dirPath := filepath.Join(here, "testdir", "filehandling")
c.Specify("The directory scanner", func() {
c.Specify("scans a directory properly", func() {
matchRegex := regexp.MustCompile(dirPath + `/subdir/.*\.log(\..*)?`)
results := ScanDirectoryForLogfiles(dirPath, matchRegex)
c.Expect(len(results), gs.Equals, 3)
})
c.Specify("scans a directory with a bad regexp", func() {
matchRegex := regexp.MustCompile(dirPath + "/subdir/.*.logg(.*)?")
results := ScanDirectoryForLogfiles(dirPath, matchRegex)
c.Expect(len(results), gs.Equals, 0)
})
})
c.Specify("Populating logfile with match parts", func() {
logfile := Logfile{}
c.Specify("works without errors", func() {
subexpNames := []string{"MonthName", "LogNumber"}
matches := []string{"October", "24"}
translation := make(SubmatchTranslationMap)
logfile.PopulateMatchParts(subexpNames, matches, translation)
c.Expect(logfile.MatchParts["MonthName"], gs.Equals, 10)
c.Expect(logfile.MatchParts["LogNumber"], gs.Equals, 24)
})
c.Specify("works with bad month name", func() {
subexpNames := []string{"MonthName", "LogNumber"}
matches := []string{"Octoberrr", "24"}
translation := make(SubmatchTranslationMap)
err := logfile.PopulateMatchParts(subexpNames, matches, translation)
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "Unable to locate month name: Octoberrr")
})
c.Specify("works with missing value in submatch translation map", func() {
subexpNames := []string{"MonthName", "LogNumber"}
matches := []string{"October", "24"}
translation := make(SubmatchTranslationMap)
translation["LogNumber"] = make(MatchTranslationMap)
translation["LogNumber"]["23"] = 22
translation["LogNumber"]["999"] = 999 // Non-"missing" submatches must be len > 1.
err := logfile.PopulateMatchParts(subexpNames, matches, translation)
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals, "Value '24' not found in translation map 'LogNumber'.")
})
c.Specify("works with custom value in submatch translation map", func() {
subexpNames := []string{"MonthName", "LogNumber"}
matches := []string{"October", "24"}
translation := make(SubmatchTranslationMap)
translation["LogNumber"] = make(MatchTranslationMap)
translation["LogNumber"]["24"] = 2
translation["LogNumber"]["999"] = 999 // Non-"missing" submatches must be len > 1.
logfile.PopulateMatchParts(subexpNames, matches, translation)
c.Expect(logfile.MatchParts["MonthName"], gs.Equals, 10)
c.Expect(logfile.MatchParts["LogNumber"], gs.Equals, 2)
})
})
c.Specify("Populating logfiles with match parts", func() {
translation := make(SubmatchTranslationMap)
matchRegex := regexp.MustCompile(dirPath + `/subdir/.*\.log\.?(?P<FileNumber>.*)?`)
logfiles := ScanDirectoryForLogfiles(dirPath, matchRegex)
c.Specify("is populated", func() {
logfiles.PopulateMatchParts(matchRegex, translation)
c.Expect(len(logfiles), gs.Equals, 3)
c.Expect(logfiles[0].MatchParts["FileNumber"], gs.Equals, -1)
c.Expect(logfiles[1].MatchParts["FileNumber"], gs.Equals, 1)
c.Expect(logfiles[1].StringMatchParts["FileNumber"], gs.Equals, "1")
})
c.Specify("returns errors", func() {
translation["FileNumber"] = make(MatchTranslationMap)
translation["FileNumber"]["23"] = 22
translation["FileNumber"]["999"] = 999 // Non-"missing" submatches must be len > 1.
err := logfiles.PopulateMatchParts(matchRegex, translation)
c.Assume(err, gs.Not(gs.IsNil))
c.Expect(len(logfiles), gs.Equals, 3)
})
})
c.Specify("Sorting logfiles", func() {
translation := make(SubmatchTranslationMap)
matchRegex := regexp.MustCompile(dirPath + `/subdir/.*\.log\.?(?P<FileNumber>.*)?`)
logfiles := ScanDirectoryForLogfiles(dirPath, matchRegex)
c.Specify("with no 'missing' translation value", func() {
err := logfiles.PopulateMatchParts(matchRegex, translation)
c.Assume(err, gs.IsNil)
c.Expect(len(logfiles), gs.Equals, 3)
c.Specify("can be sorted newest to oldest", func() {
byp := ByPriority{Logfiles: logfiles, Priority: []string{"FileNumber"}}
sort.Sort(byp)
c.Expect(logfiles[0].MatchParts["FileNumber"], gs.Equals, -1)
c.Expect(logfiles[1].MatchParts["FileNumber"], gs.Equals, 1)
})
c.Specify("can be sorted oldest to newest", func() {
byp := ByPriority{Logfiles: logfiles, Priority: []string{"^FileNumber"}}
sort.Sort(byp)
c.Expect(logfiles[0].MatchParts["FileNumber"], gs.Equals, 2)
c.Expect(logfiles[1].MatchParts["FileNumber"], gs.Equals, 1)
})
})
c.Specify("with 'missing' translation value", func() {
translation["FileNumber"] = make(MatchTranslationMap)
translation["FileNumber"]["missing"] = 5
err := logfiles.PopulateMatchParts(matchRegex, translation)
c.Assume(err, gs.IsNil)
c.Expect(len(logfiles), gs.Equals, 3)
c.Specify("honors 'missing' translation value", func() {
byp := ByPriority{Logfiles: logfiles, Priority: []string{"FileNumber"}}
sort.Sort(byp)
c.Expect(logfiles[0].MatchParts["FileNumber"], gs.Equals, 1)
c.Expect(logfiles[1].MatchParts["FileNumber"], gs.Equals, 2)
c.Expect(logfiles[2].MatchParts["FileNumber"], gs.Equals, 5)
})
})
})
c.Specify("Sorting out a directory of access/error logs", func() {
translation := make(SubmatchTranslationMap)
matchRegex := regexp.MustCompile(dirPath +
`/(?P<Year>\d+)/(?P<Month>\d+)/(?P<Type>\w+)\.log(\.(?P<Seq>\d+))?`)
logfiles := ScanDirectoryForLogfiles(dirPath, matchRegex)
err := logfiles.PopulateMatchParts(matchRegex, translation)
c.Assume(err, gs.IsNil)
c.Expect(len(logfiles), gs.Equals, 52)
c.Specify("can result in multiple logfile streams", func() {
mfs := FilterMultipleStreamFiles(logfiles, []string{"Type"})
c.Expect(len(mfs), gs.Equals, 2)
access, ok := mfs["access"]
c.Assume(ok, gs.IsTrue)
c.Expect(len(access), gs.Equals, 26)
error, ok := mfs["error"]
c.Assume(ok, gs.IsTrue)
c.Expect(len(error), gs.Equals, 26)
c.Specify("can be individually sorted properly by access", func() {
byp := ByPriority{Logfiles: mfs["access"], Priority: []string{"Year", "Month", "^Seq"}}
sort.Sort(byp)
lf := mfs["access"]
c.Expect(lf[0].FileName, gs.Equals, dirPath+"/2010/05/access.log.3")
c.Expect(lf[len(lf)-1].FileName, gs.Equals, dirPath+"/2013/08/access.log")
})
c.Specify("can be individually sorted properly by error", func() {
byp := ByPriority{Logfiles: mfs["error"], Priority: []string{"Year", "Month", "^Seq"}}
sort.Sort(byp)
lf := mfs["error"]
c.Expect(lf[0].FileName, gs.Equals, dirPath+"/2010/07/error.log.2")
c.Expect(lf[len(lf)-1].FileName, gs.Equals, dirPath+"/2013/08/error.log")
})
})
c.Specify("Can result in multiple logfile streams with a prefix", func() {
mfs := FilterMultipleStreamFiles(logfiles, []string{"website-", "Type"})
c.Expect(len(mfs), gs.Equals, 2)
access, ok := mfs["website-access"]
c.Assume(ok, gs.IsTrue)
c.Expect(len(access), gs.Equals, 26)
error, ok := mfs["website-error"]
c.Assume(ok, gs.IsTrue)
c.Expect(len(error), gs.Equals, 26)
})
})
}
func StatAccumInputSpec(c gs.Context) {
t := &ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
c.Specify("A StatAccumInput", func() {
statAccumInput := StatAccumInput{}
config := statAccumInput.ConfigStruct().(*StatAccumInputConfig)
pConfig := NewPipelineConfig(nil)
statAccumInput.pConfig = pConfig
c.Specify("ticker interval is zero", func() {
config.TickerInterval = 0
err := statAccumInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
expected := "TickerInterval must be greater than 0."
c.Expect(err.Error(), gs.Equals, expected)
})
c.Specify("validates that data is emitted", func() {
config.EmitInPayload = false
err := statAccumInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
expected := "One of either `EmitInPayload` or `EmitInFields` must be set to true."
c.Expect(err.Error(), gs.Equals, expected)
})
c.Specify("that is started", func() {
ith := new(InputTestHelper)
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
tickChan := make(chan time.Time)
var inputStarted sync.WaitGroup
runErrChan := make(chan error)
startInput := func() {
inputStarted.Add(1)
// A call to Ticker() is the last step in the input's startup before
// it's ready to process packs. Any tests that need to pause until
// this happens can use `inputStarted.Wait()`.
ith.MockInputRunner.EXPECT().Ticker().Do(func() {
inputStarted.Done()
}).Return(tickChan)
go func() {
err := statAccumInput.Run(ith.MockInputRunner, ith.MockHelper)
runErrChan <- err
}()
}
ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
// Need one of these for every message delivered.
ith.MockInputRunner.EXPECT().Name().Return("StatAccumInput")
deliverCall := ith.MockInputRunner.EXPECT().Deliver(ith.Pack)
var deliverCalled sync.WaitGroup
deliverCalled.Add(1)
deliverCall.Do(func(pack *PipelinePack) {
deliverCalled.Done()
})
c.Specify("using normal namespaces", func() {
config.EmitInFields = true
config.EmitInPayload = false
err := statAccumInput.Init(config)
c.Expect(err, gs.IsNil)
finalizeSendingStats := func() (*message.Message, error) {
close(statAccumInput.statChan)
err := <-runErrChan
return ith.Pack.Message, err
}
sendTimer := func(key string, vals ...int) {
for _, v := range vals {
statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "ms", float32(1)}
}
}
sendCounter := func(key string, vals ...int) {
for _, v := range vals {
statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "c", float32(1)}
}
}
sendGauge := func(key string, vals ...int) {
for _, v := range vals {
statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "g", float32(1)}
}
}
drainStats := func() {
ok := true
for ok {
if len(statAccumInput.statChan) > 0 {
time.Sleep(100 * time.Millisecond)
} else {
ok = false
}
}
}
validateValueAtKey := func(msg *message.Message, key string, value interface{}) {
fieldValue, ok := msg.GetFieldValue(key)
if !ok {
log.Printf("%s field is missing from the message\n", key)
}
c.Expect(ok, gs.IsTrue)
if fieldValue != value {
log.Printf("%s should be %v is %v\n", key, value, fieldValue)
}
c.Expect(fieldValue, gs.Equals, value)
}
c.Specify("emits timer with correct prefixes", func() {
startInput()
sendTimer("sample.timer", 10, 10, 20, 20)
sendTimer("sample2.timer", 10)
msg, err := finalizeSendingStats()
c.Assume(err, gs.Equals, nil)
validateValueAtKey(msg, "stats.timers.sample.timer.count", int64(4))
validateValueAtKey(msg, "stats.timers.sample.timer.count_ps", 0.4)
validateValueAtKey(msg, "stats.timers.sample.timer.mean", 15.0)
validateValueAtKey(msg, "stats.timers.sample.timer.lower", 10.0)
validateValueAtKey(msg, "stats.timers.sample.timer.upper", 20.0)
validateValueAtKey(msg, "stats.timers.sample.timer.sum", 60.0)
validateValueAtKey(msg, "stats.timers.sample.timer.mean_90", 15.0)
validateValueAtKey(msg, "stats.timers.sample.timer.upper_90", 20.0)
validateValueAtKey(msg, "stats.timers.sample2.timer.count", int64(1))
validateValueAtKey(msg, "stats.timers.sample2.timer.count_ps", 0.1)
validateValueAtKey(msg, "stats.timers.sample2.timer.mean", 10.0)
validateValueAtKey(msg, "stats.timers.sample2.timer.lower", 10.0)
validateValueAtKey(msg, "stats.timers.sample2.timer.upper", 10.0)
validateValueAtKey(msg, "stats.timers.sample2.timer.sum", 10.0)
validateValueAtKey(msg, "stats.timers.sample2.timer.mean_90", 10.0)
validateValueAtKey(msg, "stats.timers.sample2.timer.upper_90", 10.0)
validateValueAtKey(msg, "stats.statsd.numStats", int64(2))
})
c.Specify("emits counters with correct prefixes", func() {
startInput()
sendCounter("sample.cnt", 1, 2, 3, 4, 5)
sendCounter("sample2.cnt", 159, 951)
msg, err := finalizeSendingStats()
c.Assume(err, gs.IsNil)
validateValueAtKey(msg, "stats.counters.sample.cnt.count", int64(15))
validateValueAtKey(msg, "stats.counters.sample.cnt.rate", 1.5)
validateValueAtKey(msg, "stats.counters.sample2.cnt.count", int64(1110))
validateValueAtKey(msg, "stats.counters.sample2.cnt.rate", 1110.0/float64(config.TickerInterval))
})
c.Specify("emits gauge with correct prefixes", func() {
startInput()
sendGauge("sample.gauge", 1, 2)
sendGauge("sample2.gauge", 1, 2, 3, 4, 5)
msg, err := finalizeSendingStats()
c.Assume(err, gs.IsNil)
validateValueAtKey(msg, "stats.gauges.sample.gauge", float64(2))
validateValueAtKey(msg, "stats.gauges.sample2.gauge", float64(5))
})
c.Specify("emits correct statsd.numStats count", func() {
startInput()
sendGauge("sample.gauge", 1, 2)
sendGauge("sample2.gauge", 1, 2)
sendCounter("sample.cnt", 1, 2, 3, 4, 5)
sendCounter("sample2.cnt", 159, 951)
sendTimer("sample.timer", 10, 10, 20, 20)
sendTimer("sample2.timer", 10, 20)
msg, err := finalizeSendingStats()
c.Assume(err, gs.IsNil)
validateValueAtKey(msg, "stats.statsd.numStats", int64(6))
})
c.Specify("emits proper idle stats", func() {
startInput()
inputStarted.Wait()
sendGauge("sample.gauge", 1, 2)
sendCounter("sample.cnt", 1, 2, 3, 4, 5)
sendTimer("sample.timer", 10, 10, 20, 20)
drainStats()
tickChan <- time.Now()
deliverCalled.Wait()
ith.Pack.Recycle(nil)
ith.PackSupply <- ith.Pack
ith.MockInputRunner.EXPECT().Name().Return("StatAccumInput")
ith.MockInputRunner.EXPECT().Deliver(ith.Pack)
msg, err := finalizeSendingStats()
c.Assume(err, gs.IsNil)
validateValueAtKey(msg, "stats.gauges.sample.gauge", float64(2))
validateValueAtKey(msg, "stats.counters.sample.cnt.count", int64(0))
validateValueAtKey(msg, "stats.timers.sample.timer.count", int64(0))
validateValueAtKey(msg, "stats.statsd.numStats", int64(3))
})
c.Specify("omits idle stats", func() {
config.DeleteIdleStats = true
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
inputStarted.Wait() // Can't flush until the input has started.
sendGauge("sample.gauge", 1, 2)
sendCounter("sample.cnt", 1, 2, 3, 4, 5)
sendTimer("sample.timer", 10, 10, 20, 20)
drainStats()
tickChan <- time.Now()
deliverCalled.Wait()
sendTimer("sample2.timer", 10, 20)
drainStats()
ith.Pack.Recycle(nil)
ith.PackSupply <- ith.Pack
ith.MockInputRunner.EXPECT().Name().Return("StatAccumInput")
ith.MockInputRunner.EXPECT().Deliver(ith.Pack)
msg, err := finalizeSendingStats()
c.Assume(err, gs.IsNil)
validateValueAtKey(msg, "stats.statsd.numStats", int64(1))
})
})
c.Specify("using legacy namespaces", func() {
config.LegacyNamespaces = true
statName := "sample.stat"
statVal := int64(303)
testStat := Stat{statName, strconv.Itoa(int(statVal)), "c", float32(1)}
validateMsgFields := func(msg *message.Message) {
c.Expect(len(msg.Fields), gs.Equals, 4)
// timestamp
_, ok := msg.GetFieldValue("timestamp")
c.Expect(ok, gs.IsTrue)
var tmp interface{}
var intTmp int64
// stats.sample.stat
tmp, ok = msg.GetFieldValue("stats." + statName)
c.Expect(ok, gs.IsTrue)
intTmp, ok = tmp.(int64)
c.Expect(ok, gs.IsTrue)
c.Expect(intTmp, gs.Equals, int64(30))
// stats_counts.sample.stat
tmp, ok = msg.GetFieldValue("stats_counts." + statName)
c.Expect(ok, gs.IsTrue)
intTmp, ok = tmp.(int64)
c.Expect(ok, gs.IsTrue)
c.Expect(intTmp, gs.Equals, statVal)
// statsd.numStats
tmp, ok = msg.GetFieldValue("statsd.numStats")
c.Expect(ok, gs.IsTrue)
intTmp, ok = tmp.(int64)
c.Expect(ok, gs.IsTrue)
c.Expect(intTmp, gs.Equals, int64(1))
}
validateMsgPayload := func(msg *message.Message) {
lines := strings.Split(msg.GetPayload(), "\n")
c.Expect(len(lines), gs.Equals, 4)
c.Expect(lines[3], gs.Equals, "")
var timestamp string
for i := 0; i < 3; i++ {
line := strings.Split(lines[i], " ")
c.Expect(len(line), gs.Equals, 3)
switch i {
case 0:
c.Expect(line[0], gs.Equals, "stats."+statName)
c.Expect(line[1], gs.Equals, "30.300000")
timestamp = line[2]
case 1:
c.Expect(line[0], gs.Equals, "stats_counts."+statName)
c.Expect(line[1], gs.Equals, strconv.Itoa(int(statVal)))
c.Expect(line[2], gs.Equals, timestamp)
case 2:
c.Expect(line[0], gs.Equals, "statsd.numStats")
c.Expect(line[1], gs.Equals, "1")
c.Expect(line[2], gs.Equals, timestamp)
}
}
expected := strings.Join(lines, "\n")
c.Expect(msg.GetPayload(), gs.Equals, expected)
}
c.Specify("emits data in payload by default", func() {
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
statAccumInput.statChan <- testStat
close(statAccumInput.statChan)
err = <-runErrChan
c.Expect(err, gs.IsNil)
validateMsgPayload(ith.Pack.Message)
})
c.Specify("emits data in fields when specified", func() {
config.EmitInFields = true
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
statAccumInput.statChan <- testStat
close(statAccumInput.statChan)
err = <-runErrChan
c.Expect(err, gs.IsNil)
validateMsgFields(ith.Pack.Message)
validateMsgPayload(ith.Pack.Message)
})
c.Specify("omits data in payload when specified", func() {
config.EmitInPayload = false
config.EmitInFields = true
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
statAccumInput.statChan <- testStat
close(statAccumInput.statChan)
err = <-runErrChan
c.Expect(err, gs.IsNil)
validateMsgFields(ith.Pack.Message)
c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "")
})
c.Specify("honors time ticker to flush", func() {
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
inputStarted.Wait()
statAccumInput.statChan <- testStat
// Sleep until the stat is processed.
for len(statAccumInput.statChan) > 0 {
time.Sleep(50)
}
tickChan <- time.Now()
// Don't try to validate our message payload until Deliver has
// been called.
deliverCalled.Wait()
validateMsgPayload(ith.Pack.Message)
// Prep pack and EXPECTS for the close.
ith.PackSupply <- ith.Pack
ith.MockInputRunner.EXPECT().Name().Return("StatAccumInput")
ith.MockInputRunner.EXPECT().Deliver(ith.Pack)
close(statAccumInput.statChan)
err = <-runErrChan
c.Expect(err, gs.IsNil)
})
c.Specify("correctly processes timers", func() {
sendTimer := func(vals ...int) {
for _, v := range vals {
statAccumInput.statChan <- Stat{"sample.timer", strconv.Itoa(int(v)),
"ms", float32(1)}
}
}
config.EmitInFields = true
err := statAccumInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
sendTimer(220, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100)
close(statAccumInput.statChan)
err = <-runErrChan
c.Expect(err, gs.IsNil)
msg := ith.Pack.Message
getVal := func(token string) float64 {
tmp, ok := msg.GetFieldValue("stats.timers.sample.timer." + token)
c.Expect(ok, gs.IsTrue)
val, ok := tmp.(float64)
c.Expect(ok, gs.IsTrue)
return val
}
c.Expect(getVal("upper"), gs.Equals, 220.0)
c.Expect(getVal("lower"), gs.Equals, 10.0)
c.Expect(getVal("mean"), gs.Equals, 70.0)
c.Expect(getVal("upper_90"), gs.Equals, 100.0)
c.Expect(getVal("mean_90"), gs.Equals, 55.0)
tmp, ok := msg.GetFieldValue("stats.timers.sample.timer.count")
c.Expect(ok, gs.IsTrue)
intTmp, ok := tmp.(int64)
c.Expect(ok, gs.IsTrue)
c.Expect(intTmp, gs.Equals, int64(11))
})
})
})
})
}
