Example #1
0
func DecodersSpec(c gospec.Context) {
	msg := getTestMessage()

	c.Specify("A JsonDecoder", func() {
		var fmtString = `{"type":"%s","timestamp":%s,"logger":"%s","severity":%d,"payload":"%s","fields":%s,"env_version":"%s","metlog_pid":%d,"metlog_hostname":"%s"}`
		timestampJson, err := json.Marshal(msg.Timestamp)
		fieldsJson, err := json.Marshal(msg.Fields)
		c.Assume(err, gs.IsNil)
		jsonString := fmt.Sprintf(fmtString, msg.Type,
			timestampJson, msg.Logger, msg.Severity, msg.Payload,
			fieldsJson, msg.Env_version, msg.Pid, msg.Hostname)

		pipelinePack := getTestPipelinePack()
		pipelinePack.MsgBytes = []byte(jsonString)
		jsonDecoder := new(JsonDecoder)

		c.Specify("can decode a JSON message", func() {
			err := jsonDecoder.Decode(pipelinePack)
			c.Expect(pipelinePack.Message, gs.Equals, msg)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("returns `fields` as a map", func() {
			jsonDecoder.Decode(pipelinePack)
			c.Expect(pipelinePack.Message.Fields["foo"], gs.Equals, "bar")
		})

		c.Specify("returns an error for bogus JSON", func() {
			badJson := fmt.Sprint("{{", jsonString)
			pipelinePack.MsgBytes = []byte(badJson)
			err := jsonDecoder.Decode(pipelinePack)
			c.Expect(err, gs.Not(gs.IsNil))
			c.Expect(pipelinePack.Message.Timestamp.IsZero(), gs.IsTrue)
		})
	})

	c.Specify("A MsgPackDecoder", func() {
		msg := getTestMessage()
		encoded, err := msgpack.Marshal(msg)
		c.Assume(err, gs.IsNil)

		decoder := new(MsgPackDecoder)
		decoder.Init(nil)
		pack := getTestPipelinePack()

		c.Specify("decodes a msgpack message", func() {
			pack.MsgBytes = encoded
			err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(pack.Message, gs.Equals, msg)
		})

		c.Specify("returns an error for bunk encoding", func() {
			bunk := append([]byte{0, 0, 0}, encoded...)
			pack.MsgBytes = bunk
			err := decoder.Decode(pack)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
Example #2
0
func ProtobufDecoderSpec(c gospec.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	msg := ts.GetTestMessage()
	config := NewPipelineConfig(nil) // Initializes globals.

	c.Specify("A ProtobufDecoder", func() {
		encoded, err := proto.Marshal(msg)
		c.Assume(err, gs.IsNil)
		pack := NewPipelinePack(config.inputRecycleChan)
		decoder := new(ProtobufDecoder)
		decoder.sampleDenominator = 1000 // Since we don't call decoder.Init().

		c.Specify("decodes a protobuf message", func() {
			pack.MsgBytes = encoded
			_, err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(pack.Message, gs.Equals, msg)
			v, ok := pack.Message.GetFieldValue("foo")
			c.Expect(ok, gs.IsTrue)
			c.Expect(v, gs.Equals, "bar")
		})

		c.Specify("returns an error for bunk encoding", func() {
			bunk := append([]byte{0, 0, 0}, encoded...)
			pack.MsgBytes = bunk
			_, err := decoder.Decode(pack)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
Example #3
0
func PrimitiveDecodeStrictSpec(c gs.Context) {
	var md MetaData
	var err error

	var tomlBlob = `
ranking = ["Springsteen", "J Geils"]

[bands.Springsteen]
type = "ignore_this"
started = 1973
albums = ["Greetings", "WIESS", "Born to Run", "Darkness"]
not_albums = ["Greetings", "WIESS", "Born to Run", "Darkness"]

[bands.J Geils]
started = 1970
albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"]
`

	type band struct {
		Started int
		Albums  []string
	}

	type classics struct {
		Ranking []string
		Bands   map[string]Primitive
	}

	// Do the initial decode. Reflection is delayed on Primitive values.
	var music classics
	md, err = Decode(tomlBlob, &music)
	c.Assume(err, gs.IsNil)

	// MetaData still includes information on Primitive values.
	c.Assume(md.IsDefined("bands", "Springsteen"), gs.IsTrue)

	ignore_type := map[string]interface{}{"type": true}
	// Decode primitive data into Go values.
	for _, artist := range music.Ranking {
		// A band is a primitive value, so we need to decode it to get a
		// real `band` value.
		primValue := music.Bands[artist]

		var aBand band

		err = PrimitiveDecodeStrict(primValue, &aBand, ignore_type)
		if artist == "Springsteen" {
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), gs.Equals, "Configuration contains key [not_albums] which doesn't exist in struct")
			c.Assume(1973, gs.Equals, aBand.Started)
		} else {
			c.Expect(err, gs.IsNil)
			c.Assume(1970, gs.Equals, aBand.Started)
		}

	}
}
Example #4
0
func UdpInputSpecFailure(c gs.Context) {
	udpInput := UdpInput{}
	err := udpInput.Init(&UdpInputConfig{Net: "tcp", Address: "localhost:55565",
		Decoder:    "ProtobufDecoder",
		ParserType: "message.proto"})
	c.Assume(err, gs.Not(gs.IsNil))
	c.Assume(err.Error(), gs.Equals, "ResolveUDPAddr failed: unknown network tcp\n")

}
Example #5
0
func TcpInputSpecFailure(c gs.Context) {
	tcpInput := TcpInput{}
	err := tcpInput.Init(&TcpInputConfig{Net: "udp", Address: "localhost:55565",
		Decoder:    "ProtobufDecoder",
		ParserType: "message.proto"})
	c.Assume(err, gs.Not(gs.IsNil))
	c.Assume(err.Error(), gs.Equals, "ListenTCP failed: unknown network udp\n")

}
Example #6
0
func UdpInputSpecFailure(c gs.Context) {
	udpInput := UdpInput{}
	err := udpInput.Init(&UdpInputConfig{
		Net:     "tcp",
		Address: "localhost:55565",
	})
	c.Assume(err, gs.Not(gs.IsNil))
	c.Assume(err.Error(), gs.Equals, "ResolveUDPAddr failed: unknown network tcp\n")

}
Example #7
0
func TcpInputSpecFailure(c gs.Context) {
	tcpInput := TcpInput{}
	err := tcpInput.Init(&TcpInputConfig{
		Net:     "udp",
		Address: "localhost:55565",
	})
	c.Assume(err, gs.Not(gs.IsNil))
	c.Assume(err.Error(), gs.Equals, "ResolveTCPAddress failed: unknown network udp\n")

}
Example #8
0
func MessageEqualsSpec(c gospec.Context) {
	msg0 := getTestMessage()
	msg1Real := *msg0
	msg1 := &msg1Real

	c.Specify("Messages are equal", func() {
		c.Expect(msg0, gs.Equals, msg1)
	})

	c.Specify("Messages w/ diff int values are not equal", func() {
		msg1.Severity--
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ diff string values are not equal", func() {
		msg1.Payload = "Something completely different"
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ diff maps are not equal", func() {
		msg1.Fields = map[string]interface{}{"sna": "foo"}
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})
}
Example #9
0
func DecodeSpec(c gs.Context) {
	var val simple

	md, err := Decode(testSimple, &val)
	c.Assume(err, gs.IsNil)
	c.Assume(md.IsDefined("Annoying", "Cats", "plato"), gs.IsTrue)
	c.Assume(md.IsDefined("Cats", "Stinky"), gs.IsFalse)
	var colors = [][]string{[]string{"red", "green", "blue"},
		[]string{"cyan", "magenta", "yellow", "black"}}
	for ridx, row := range colors {
		for cidx, _ := range row {
			c.Assume(val.Colors[ridx][cidx], gs.Equals, colors[ridx][cidx])
		}
	}
	c.Assume(val, gs.Not(gs.IsNil))
}
Example #10
0
func DashboardOutputSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	c.Specify("A FileOutput", func() {
		dashboardOutput := new(DashboardOutput)

		config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
		c.Specify("Init halts if basedirectory is not writable", func() {
			tmpdir := path.Join(os.TempDir(), "tmpdir")
			err := os.MkdirAll(tmpdir, 0400)
			c.Assume(err, gs.IsNil)
			config.WorkingDirectory = tmpdir
			err = dashboardOutput.Init(config)
			c.Assume(err, gs.Not(gs.IsNil))
		})
	})
}
Example #11
0
func DashboardOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	NewPipelineConfig(nil) // Needed for side effect of setting up Globals :P

	if runtime.GOOS != "windows" {
		c.Specify("A DashboardOutput", func() {
			dashboardOutput := new(DashboardOutput)

			config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
			c.Specify("Init halts if basedirectory is not writable", func() {
				tmpdir := filepath.Join(os.TempDir(), "tmpdir")
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				config.WorkingDirectory = tmpdir
				err = dashboardOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
			})
		})
	}
}
Example #12
0
func JsonPathSpec(c gs.Context) {
	c.Specify("JsonPath can read data", func() {
		var s = `{
	"foo": {
		"bar": [
			{
				"baz": "こんにちわ世界",
				"noo": "aaa"
			},
			{
				"maz": "123",
				"moo": 256,
				"muux": 2.10
			}
		],
		"boo": {
			"bag": true,
			"bug": false
		}
	}
}
`
		var err error
		var json_path *JsonPath
		var result interface{}

		json_path = new(JsonPath)
		err = json_path.SetJsonText(s)
		c.Expect(err, gs.IsNil)

		result, err = json_path.Find("$.foo.bar[0].baz")
		c.Expect(err, gs.IsNil)
		c.Expect(result, gs.Equals, "こんにちわ世界")

		result, err = json_path.Find("$.foo.bar[0].noo")
		c.Expect(err, gs.IsNil)
		c.Expect(result, gs.Equals, "aaa")

		result, err = json_path.Find("$.foo.bar[1].maz")
		c.Expect(err, gs.IsNil)
		c.Expect(result, gs.Equals, "123")

		result, err = json_path.Find("$.foo.bar[1].moo")
		c.Expect(err, gs.IsNil)
		c.Expect(result, gs.Equals, "256")

		result, err = json_path.Find("$.foo.bar[1].muux")
		c.Expect(err, gs.IsNil)
		c.Expect(result, gs.Equals, "2.10")

		result, err = json_path.Find("$.foo.boo.bag")
		c.Expect(err, gs.IsNil)
		c.Expect(result, gs.Equals, "true")

		result, err = json_path.Find("$.foo.boo.bug")
		c.Expect(err, gs.IsNil)
		c.Expect(result, gs.Equals, "false")

		result, err = json_path.Find("$.foo.bar[99].baz")
		c.Expect(err, gs.Not(gs.IsNil))

		result, err = json_path.Find("$.badpath")
		c.Expect(err, gs.Not(gs.IsNil))

		result, err = json_path.Find("badpath")
		c.Expect(err, gs.Not(gs.IsNil))

		result, err = json_path.Find("$.foo.bar.3428")
		c.Expect(err, gs.Not(gs.IsNil))

		expected_data := `[{"baz":"こんにちわ世界","noo":"aaa"},{"maz":"123","moo":256,"muux":2.10}]`
		result_data, err := json_path.Find("$.foo.bar")
		c.Expect(err, gs.IsNil)
		c.Expect(result_data, gs.Equals, expected_data)

	})

	c.Specify("JsonPath doesn't crash on nil data", func() {
		var err error
		var json_path *JsonPath

		json_path = new(JsonPath)
		err = json_path.SetJsonText("")
		c.Expect(err, gs.Not(gs.IsNil))

		// Searches should return an error
		result, err := json_path.Find("$.foo.bar.3428")
		c.Expect(err, gs.Not(gs.IsNil))
		c.Expect(err.Error(), gs.Equals, "JSON data is nil")
		c.Expect(result, gs.Equals, "")
	})

	c.Specify("JsonPath handles arrays at top level", func() {
		var err error
		var json_path *JsonPath

		json_path = new(JsonPath)
		err = json_path.SetJsonText(`["foo"]`)
		c.Expect(err, gs.IsNil)

		// Searches should return an error
		result, err := json_path.Find("$.[0]")
		c.Expect(err, gs.IsNil)
		c.Expect(result, gs.Equals, "foo")
	})

	c.Specify("JsonPath handles invalid doubly encoded JSON gracefully", func() {
		s := `{
            "request":{
                "parameters":{
                    "invites":"[{\"inviteUserId\":\"123\",\"email\":\"[email protected]\",\"phone\":\"123\",\"name\":\"John
                    Doe\"}]",
                    "feature":"0"
                }
            }
        }`

		var err error
		var json_path *JsonPath

		json_path = new(JsonPath)
		err = json_path.SetJsonText(s)
		c.Expect(err, gs.Not(gs.IsNil))

		// Searches should return an error
		result, err := json_path.Find("$.[0]")
		c.Expect(err, gs.Not(gs.IsNil))
		c.Expect(err.Error(), gs.Equals, "JSON data is nil")
		c.Expect(result, gs.Equals, "")
	})

}
Example #13
0
func DashboardOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := pipeline.NewPipelineConfig(nil)

	dashboardOutput := new(DashboardOutput)
	dashboardOutput.pConfig = pConfig

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)

	errChan := make(chan error, 1)

	startOutput := func() {
		go func() {
			errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
		}()
	}

	if runtime.GOOS != "windows" {
		c.Specify("A DashboardOutput", func() {

			tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
			c.Assume(err, gs.IsNil)
			config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
			config.WorkingDirectory = tmpdir

			c.Specify("Init halts if basedirectory is not writable", func() {
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)
				err = dashboardOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
			})

			c.Specify("that is running", func() {
				startedChan := make(chan bool, 1)
				defer close(startedChan)
				ts := httptest.NewUnstartedServer(nil)

				dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
					ts.Start()
					startedChan <- true
					return nil
				}

				ticker := make(chan time.Time)
				inChan := make(chan *pipeline.PipelinePack, 1)
				recycleChan := make(chan *pipeline.PipelinePack, 1)
				pack := pipeline.NewPipelinePack(recycleChan)
				pack.Message = pipeline_ts.GetTestMessage()

				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)

				err := os.MkdirAll(tmpdir, 0700)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)

				dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					// Noop
				})

				c.Specify("sets custom http headers", func() {
					config.Headers = http.Header{
						"One":  []string{"two", "three"},
						"Four": []string{"five", "six", "seven"},
					}
					err = dashboardOutput.Init(config)
					c.Assume(err, gs.IsNil)
					ts.Config = dashboardOutput.server

					startOutput()

					inChan <- pack
					<-startedChan
					resp, err := http.Get(ts.URL)
					c.Assume(err, gs.IsNil)
					resp.Body.Close()
					c.Assume(resp.StatusCode, gs.Equals, 200)

					// Verify headers are there
					eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
					c.Expect(eq, gs.IsTrue)
					eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
					c.Expect(eq, gs.IsTrue)
				})

				close(inChan)
				c.Expect(<-errChan, gs.IsNil)

				ts.Close()
			})
		})
	}
}
Example #14
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	pConfig := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network.
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// Set up mock helper, runner, and pack supply channel.
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: pConfig}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs.
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
				ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
					ith.MockSplitterRunner)
				ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
				ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
				ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

				getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
					gomock.Any()).Times(numLines)
				line := "boo hoo foo foo"
				getRecCall.Return(len(line), []byte(line), nil)
				getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
				getRecCall.Return(0, make([]byte, 0), io.EOF)

				deliverChan := make(chan []byte, 1)
				deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
					ith.MockDeliverer).Times(numLines)
				deliverCall.Do(func(record []byte, del Deliverer) {
					deliverChan <- record
				})

				ith.MockDeliverer.EXPECT().Done()

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				dur, _ := time.ParseDuration("5s")
				timeout := time.After(dur)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case record := <-deliverChan:
						c.Expect(string(record), gs.Equals, line)
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Example #15
0
func OutputsSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	c.Specify("A FileOutput", func() {
		fileOutput := new(FileOutput)

		tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
		tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
			tmpFileName)
		config := fileOutput.ConfigStruct().(*FileOutputConfig)
		config.Path = tmpFilePath

		msg := getTestMessage()
		pack := NewPipelinePack(pConfig.inputRecycleChan)
		pack.Message = msg
		pack.Decoded = true

		toString := func(outData interface{}) string {
			return string(*(outData.(*[]byte)))
		}

		c.Specify("correctly formats text output", func() {
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 20)

			c.Specify("by default", func() {
				fileOutput.handleMessage(pack, &outData)
				c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
			})

			c.Specify("w/ a prepended timestamp when specified", func() {
				fileOutput.prefix_ts = true
				fileOutput.handleMessage(pack, &outData)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				payload := *msg.Payload
				c.Expect(strContents, ts.StringContains, payload)
				c.Expect(strContents, ts.StringStartsWith, todayStr)
			})
		})

		c.Specify("correctly formats JSON output", func() {
			config.Format = "json"
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 200)

			c.Specify("when specified", func() {
				fileOutput.handleMessage(pack, &outData)
				msgJson, err := json.Marshal(pack.Message)
				c.Assume(err, gs.IsNil)
				c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
			})

			c.Specify("and with a timestamp", func() {
				fileOutput.prefix_ts = true
				fileOutput.handleMessage(pack, &outData)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				msgJson, err := json.Marshal(pack.Message)
				c.Assume(err, gs.IsNil)
				c.Expect(strContents, ts.StringContains, string(msgJson)+"\n")
				c.Expect(strContents, ts.StringStartsWith, todayStr)
			})
		})

		c.Specify("correctly formats protocol buffer stream output", func() {
			config.Format = "protobufstream"
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 200)

			c.Specify("when specified and timestamp ignored", func() {
				fileOutput.prefix_ts = true
				err := fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
				c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
			})
		})

		c.Specify("processes incoming messages", func() {
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			// Save for comparison.
			payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())

			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			wg.Add(1)
			go fileOutput.receiver(oth.MockOutputRunner, &wg)
			inChan <- pack
			close(inChan)
			outBatch := <-fileOutput.batchChan
			wg.Wait()
			c.Expect(string(outBatch), gs.Equals, payload)
		})

		c.Specify("Init halts if basedirectory is not writable", func() {
			tmpdir := filepath.Join(os.TempDir(), "tmpdir")
			err := os.MkdirAll(tmpdir, 0400)
			c.Assume(err, gs.IsNil)
			config.Path = tmpdir
			err = fileOutput.Init(config)
			c.Assume(err, gs.Not(gs.IsNil))
		})

		c.Specify("commits to a file", func() {
			outStr := "Write me out to the log file"
			outBytes := []byte(outStr)

			c.Specify("with default settings", func() {
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				contents, err := ioutil.ReadAll(tmpFile)
				c.Assume(err, gs.IsNil)
				c.Expect(string(contents), gs.Equals, outStr)
			})

			c.Specify("with different Perm settings", func() {
				config.Perm = "600"
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				fileInfo, err := tmpFile.Stat()
				c.Assume(err, gs.IsNil)
				fileMode := fileInfo.Mode()
				if runtime.GOOS == "windows" {
					c.Expect(fileMode.String(), ts.StringContains, "-rw-rw-rw-")
				} else {
					// 7 consecutive dashes implies no perms for group or other
					c.Expect(fileMode.String(), ts.StringContains, "-------")
				}
			})
		})
	})

	c.Specify("A TcpOutput", func() {
		tcpOutput := new(TcpOutput)
		config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
		tcpOutput.connection = ts.NewMockConn(ctrl)

		msg := getTestMessage()
		pack := NewPipelinePack(pConfig.inputRecycleChan)
		pack.Message = msg
		pack.Decoded = true

		c.Specify("correctly formats protocol buffer stream output", func() {
			outBytes := make([]byte, 0, 200)
			err := createProtobufStream(pack, &outBytes)
			c.Expect(err, gs.IsNil)

			b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
			c.Expect(bytes.Equal(b, (outBytes)[:len(b)]), gs.IsTrue)
		})

		c.Specify("writes out to the network", func() {
			inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
			inChanCall.Return(inChan)

			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
				}
				ch <- "ready"
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch // wait for server

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			outStr := "Write me out to the network"
			pack.Message.SetPayload(outStr)
			go func() {
				wg.Add(1)
				tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
			inChan <- pack

			close(inChan)
			wg.Wait() // wait for close to finish, prevents intermittent test failures

			matchBytes := make([]byte, 0, 1000)
			err = createProtobufStream(pack, &matchBytes)
			c.Expect(err, gs.IsNil)

			result = <-ch
			c.Expect(result, gs.Equals, string(matchBytes))
		})
	})

	c.Specify("Runner restarts a plugin on the first time only", func() {
		pc := new(PipelineConfig)
		var pluginGlobals PluginGlobals
		pluginGlobals.Retries = RetryOptions{
			MaxDelay:   "1us",
			Delay:      "1us",
			MaxJitter:  "1us",
			MaxRetries: 1,
		}
		pw := &PluginWrapper{
			name:          "stoppingOutput",
			configCreator: func() interface{} { return nil },
			pluginCreator: func() interface{} { return new(StoppingOutput) },
		}
		output := new(StoppingOutput)
		pc.outputWrappers = make(map[string]*PluginWrapper)
		pc.outputWrappers["stoppingOutput"] = pw
		oRunner := NewFORunner("stoppingOutput", output, &pluginGlobals)
		var wg sync.WaitGroup
		cfgCall := oth.MockHelper.EXPECT().PipelineConfig()
		cfgCall.Return(pc)
		wg.Add(1)
		oRunner.Start(oth.MockHelper, &wg) // no panic => success
		wg.Wait()
		c.Expect(stopoutputTimes, gs.Equals, 2)
	})

	c.Specify("Runner restarts plugin and resumes feeding it", func() {
		pc := new(PipelineConfig)
		var pluginGlobals PluginGlobals
		pluginGlobals.Retries = RetryOptions{
			MaxDelay:   "1us",
			Delay:      "1us",
			MaxJitter:  "1us",
			MaxRetries: 4,
		}
		pw := &PluginWrapper{
			name:          "stoppingresumeOutput",
			configCreator: func() interface{} { return nil },
			pluginCreator: func() interface{} { return new(StopResumeOutput) },
		}
		output := new(StopResumeOutput)
		pc.outputWrappers = make(map[string]*PluginWrapper)
		pc.outputWrappers["stoppingresumeOutput"] = pw
		oRunner := NewFORunner("stoppingresumeOutput", output, &pluginGlobals)
		var wg sync.WaitGroup
		cfgCall := oth.MockHelper.EXPECT().PipelineConfig()
		cfgCall.Return(pc)
		wg.Add(1)
		oRunner.Start(oth.MockHelper, &wg) // no panic => success
		wg.Wait()
		c.Expect(stopresumerunTimes, gs.Equals, 3)
		c.Expect(len(stopresumeHolder), gs.Equals, 2)
		c.Expect(stopresumeHolder[1], gs.Equals, "woot")
		c.Expect(oRunner.retainPack, gs.IsNil)
	})
}
Example #16
0
func FileOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	c.Specify("A FileOutput", func() {
		fileOutput := new(FileOutput)

		tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
		tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
			tmpFileName)
		config := fileOutput.ConfigStruct().(*FileOutputConfig)
		config.Path = tmpFilePath

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		toString := func(outData interface{}) string {
			return string(*(outData.(*[]byte)))
		}

		c.Specify("correctly formats text output", func() {
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 20)

			c.Specify("by default", func() {
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
			})

			c.Specify("w/ a prepended timestamp when specified", func() {
				fileOutput.prefix_ts = true
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				payload := *msg.Payload
				c.Expect(strContents, pipeline_ts.StringContains, payload)
				c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
			})

			c.Specify("even when payload is nil", func() {
				pack.Message.Payload = nil
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				strContents := toString(&outData)
				c.Expect(strContents, gs.Equals, "\n")
			})

			c.Specify("payload is nil and with a timestamp", func() {
				pack.Message.Payload = nil
				fileOutput.prefix_ts = true
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				c.Expect(strings.HasPrefix(strContents, todayStr), gs.IsTrue)
				c.Expect(strings.HasSuffix(strContents, " \n"), gs.IsTrue)
			})
		})

		c.Specify("correctly formats JSON output", func() {
			config.Format = "json"
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 200)

			c.Specify("when specified", func() {
				fileOutput.handleMessage(pack, &outData)
				msgJson, err := json.Marshal(pack.Message)
				c.Assume(err, gs.IsNil)
				c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
			})

			c.Specify("and with a timestamp", func() {
				fileOutput.prefix_ts = true
				fileOutput.handleMessage(pack, &outData)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				msgJson, err := json.Marshal(pack.Message)
				c.Assume(err, gs.IsNil)
				c.Expect(strContents, pipeline_ts.StringContains, string(msgJson)+"\n")
				c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
			})
		})

		c.Specify("correctly formats protocol buffer stream output", func() {
			config.Format = "protobufstream"
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 200)

			c.Specify("when specified and timestamp ignored", func() {
				fileOutput.prefix_ts = true
				err := fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
				c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
			})
		})

		c.Specify("processes incoming messages", func() {
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			// Save for comparison.
			payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())

			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			wg.Add(1)
			go fileOutput.receiver(oth.MockOutputRunner, &wg)
			inChan <- pack
			close(inChan)
			outBatch := <-fileOutput.batchChan
			wg.Wait()
			c.Expect(string(outBatch), gs.Equals, payload)
		})

		c.Specify("Init halts if basedirectory is not writable", func() {
			tmpdir := filepath.Join(os.TempDir(), "tmpdir")
			err := os.MkdirAll(tmpdir, 0400)
			c.Assume(err, gs.IsNil)
			config.Path = tmpdir
			err = fileOutput.Init(config)
			c.Assume(err, gs.Not(gs.IsNil))
		})

		c.Specify("commits to a file", func() {
			outStr := "Write me out to the log file"
			outBytes := []byte(outStr)

			c.Specify("with default settings", func() {
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				contents, err := ioutil.ReadAll(tmpFile)
				c.Assume(err, gs.IsNil)
				c.Expect(string(contents), gs.Equals, outStr)
			})

			c.Specify("with different Perm settings", func() {
				config.Perm = "600"
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				fileInfo, err := tmpFile.Stat()
				c.Assume(err, gs.IsNil)
				fileMode := fileInfo.Mode()
				if runtime.GOOS == "windows" {
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
				} else {
					// 7 consecutive dashes implies no perms for group or other
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
				}
			})
		})

		c.Specify("honors folder_perm setting", func() {
			config.FolderPerm = "750"
			config.Path = filepath.Join(tmpFilePath, "subfile")
			err := fileOutput.Init(config)
			defer os.Remove(config.Path)
			c.Assume(err, gs.IsNil)

			fi, err := os.Stat(tmpFilePath)
			c.Expect(fi.IsDir(), gs.IsTrue)
			c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
		})

		c.Specify("that starts receiving w/ a flush interval", func() {
			config.FlushInterval = 100000000 // We'll trigger the timer manually.
			inChan := make(chan *PipelinePack)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			timerChan := make(chan time.Time)

			msg2 := pipeline_ts.GetTestMessage()
			pack2 := NewPipelinePack(pConfig.InputRecycleChan())
			pack2.Message = msg2

			recvWithConfig := func(config *FileOutputConfig) {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)
				wg.Add(1)
				go fileOutput.receiver(oth.MockOutputRunner, &wg)
				runtime.Gosched() // Yield so we can overwrite the timerChan.
				fileOutput.timerChan = timerChan
			}

			cleanUp := func() {
				close(inChan)
				wg.Done()
			}

			c.Specify("honors flush interval", func() {
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack
				select {
				case _ = <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}
				timerChan <- time.Now()
				select {
				case _ = <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval AND flush count", func() {
				config.FlushCount = 2
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				timerChan <- time.Now()
				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				inChan <- pack2
				runtime.Gosched()
				select {
				case <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval OR flush count", func() {
				config.FlushCount = 2
				config.FlushOperator = "OR"
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				c.Specify("when interval triggers first", func() {
					timerChan <- time.Now()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})

				c.Specify("when count triggers first", func() {
					inChan <- pack2
					runtime.Gosched()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})
			})
		})
	})
}
Example #17
0
func MatcherSpecificationSpec(c gospec.Context) {
	msg := getTestMessage()
	uuidStr := msg.GetUuidString()
	data := []byte("data")
	date := "Mon Jan 02 15:04:05 -0700 2006"
	field1, _ := NewField("bytes", data, "")
	field2, _ := NewField("int", int64(999), "")
	field2.AddValue(int64(1024))
	field3, _ := NewField("double", float64(99.9), "")
	field4, _ := NewField("bool", true, "")
	field5, _ := NewField("foo", "alternate", "")
	field6, _ := NewField("Payload", "name=test;type=web;", "")
	field7, _ := NewField("Timestamp", date, "date-time")
	field8, _ := NewField("zero", int64(0), "")
	field9, _ := NewField("string", "43", "")
	msg.AddField(field1)
	msg.AddField(field2)
	msg.AddField(field3)
	msg.AddField(field4)
	msg.AddField(field5)
	msg.AddField(field6)
	msg.AddField(field7)
	msg.AddField(field8)
	msg.AddField(field9)

	c.Specify("A MatcherSpecification", func() {
		malformed := []string{
			"",
			"bogus",
			"Type = 'test'",                                               // invalid operator
			"Pid == 'test='",                                              // Pid is not a string
			"Type == 'test' && (Severity==7 || Payload == 'Test Payload'", // missing paren
			"Invalid == 'bogus'",                                          // unknown variable name
			"Fields[]",                                                    // empty name key
			"Fields[test][]",                                              // empty field index
			"Fields[test][a]",                                             // non numeric field index
			"Fields[test][0][]",                                           // empty array index
			"Fields[test][0][a]",                                          // non numeric array index
			"Fields[test][0][0][]",                                        // extra index dimension
			"Fields[test][xxxx",                                           // unmatched bracket
			"Pid =~ /6/",                                                  // regex not allowed on numeric
			"Pid !~ /6/",                                                  // regex not allowed on numeric
			"Type =~ /test",                                               // unmatched slash
			"Type == /test/",                                              // incorrect operator
			"Type =~ 'test'",                                              // string instead of regexp
			"Type =~ /\\ytest/",                                           // invalid escape character
			"Type != 'test\"",                                             // mis matched quote types
			"Pid =~ 6",                                                    // number instead of regexp
			"NIL",                                                         // invalid use of constant
			"Type == NIL",                                                 // existence check only works on fields
			"Fields[test] > NIL",                                          // existence check only works with equals and not equals
		}

		negative := []string{
			"FALSE",
			"Type == 'test'&&(Severity==7||Payload=='Test Payload')",
			"EnvVersion == '0.9'",
			"EnvVersion != '0.8'",
			"EnvVersion > '0.9'",
			"EnvVersion >= '0.9'",
			"EnvVersion < '0.8'",
			"EnvVersion <= '0.7'",
			"Severity == 5",
			"Severity != 6",
			"Severity < 6",
			"Severity <= 5",
			"Severity > 6",
			"Severity >= 7",
			"Fields[foo] == 'ba'",
			"Fields[foo][1] == 'bar'",
			"Fields[foo][0][1] == 'bar'",
			"Fields[bool] == FALSE",
			"Type =~ /Test/",
			"Type !~ /TEST/",
			"Payload =~ /^Payload/",
			"Type == \"te'st\"",
			"Type == 'te\"st'",
			"Fields[int] =~ /999/",
			"Fields[zero] == \"0\"",
			"Fields[string] == 43",
			"Fields[int] == NIL",
			"Fields[int][0][1] == NIL",
			"Fields[missing] != NIL",
			"Type =~ /^te/",
			"Type =~ /st$/",
			"Type !~ /^TE/",
			"Type !~ /ST$/",
			"Logger =~ /./ && Type =~ /^anything/",
		}

		positive := []string{
			"TRUE",
			"(Severity == 7 || Payload == 'Test Payload') && Type == 'TEST'",
			"EnvVersion == \"0.8\"",
			"EnvVersion == '0.8'",
			"EnvVersion != '0.9'",
			"EnvVersion > '0.7'",
			"EnvVersion >= '0.8'",
			"EnvVersion < '0.9'",
			"EnvVersion <= '0.8'",
			"Hostname != ''",
			"Logger == 'GoSpec'",
			"Pid != 0",
			"Severity != 5",
			"Severity < 7",
			"Severity <= 6",
			"Severity == 6",
			"Severity > 5",
			"Severity >= 6",
			"Timestamp > 0",
			"Type != 'test'",
			"Type == 'TEST' && Severity == 6",
			"Type == 'test' && Severity == 7 || Payload == 'Test Payload'",
			"Type == 'TEST'",
			"Type == 'foo' || Type == 'bar' || Type == 'TEST'",
			fmt.Sprintf("Uuid == '%s'", uuidStr),
			"Fields[foo] == 'bar'",
			"Fields[foo][0] == 'bar'",
			"Fields[foo][0][0] == 'bar'",
			"Fields[foo][1] == 'alternate'",
			"Fields[foo][1][0] == 'alternate'",
			"Fields[foo] == 'bar'",
			"Fields[bytes] == 'data'",
			"Fields[int] == 999",
			"Fields[int][0][1] == 1024",
			"Fields[double] == 99.9",
			"Fields[bool] == TRUE",
			"Type =~ /TEST/",
			"Type !~ /bogus/",
			"Type =~ /TEST/ && Payload =~ /Payload/",
			"Fields[foo][1] =~ /alt/",
			"Fields[Payload] =~ /name=\\w+/",
			"Type =~ /(ST)/",
			"Fields[int] != NIL",
			"Fields[int][0][1] != NIL",
			"Fields[int][0][2] == NIL",
			"Fields[missing] == NIL",
			"Type =~ /^TE/",
			"Type =~ /ST$/",
			"Type !~ /^te/",
			"Type !~ /st$/",
		}

		c.Specify("malformed matcher tests", func() {
			for _, v := range malformed {
				_, err := CreateMatcherSpecification(v)
				c.Expect(err, gs.Not(gs.IsNil))
			}
		})

		c.Specify("negative matcher tests", func() {
			for _, v := range negative {
				ms, err := CreateMatcherSpecification(v)
				c.Expect(err, gs.IsNil)
				match := ms.Match(msg)
				c.Expect(match, gs.IsFalse)
			}
		})

		c.Specify("positive matcher tests", func() {
			for _, v := range positive {
				ms, err := CreateMatcherSpecification(v)
				c.Expect(err, gs.IsNil)
				match := ms.Match(msg)
				c.Expect(match, gs.IsTrue)
			}
		})
	})
}
Example #18
0
func ReportSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	chanSize := pConfig.Globals.PluginChanSize

	checkForFields := func(c gs.Context, msg *message.Message) {
		f0Val, ok := msg.GetFieldValue(f0.GetName())
		c.Expect(ok, gs.IsTrue)
		c.Expect(f0Val.(int64), gs.Equals, f0.GetValue().(int64))
		f1Val, ok := msg.GetFieldValue(f1.GetName())
		c.Expect(ok, gs.IsTrue)
		c.Expect(f1Val.(string), gs.Equals, f1.GetValue().(string))
	}

	hasChannelData := func(msg *message.Message) (ok bool) {
		capVal, _ := msg.GetFieldValue("InChanCapacity")
		lenVal, _ := msg.GetFieldValue("InChanLength")
		var i int64
		if i, ok = capVal.(int64); !ok {
			return
		}
		if ok = (i == int64(chanSize)); !ok {
			return
		}
		if i, ok = lenVal.(int64); !ok {
			return
		}
		ok = (i == int64(0))
		return
	}

	fName := "counter"
	filter := new(CounterFilter)
	fRunner := NewFORunner(fName, filter, nil, chanSize)
	var err error
	fRunner.matcher, err = NewMatchRunner("Type == ''", "", fRunner, chanSize)
	c.Assume(err, gs.IsNil)
	fRunner.matcher.inChan = make(chan *PipelinePack, chanSize)
	leakCount := 10
	fRunner.SetLeakCount(leakCount)

	iName := "stat_accum"
	input := new(StatAccumInput)
	iRunner := NewInputRunner(iName, input, nil, false)

	c.Specify("`PopulateReportMsg`", func() {
		msg := ts.GetTestMessage()

		c.Specify("w/ a filter", func() {
			err := PopulateReportMsg(fRunner, msg)
			c.Assume(err, gs.IsNil)

			c.Specify("invokes `ReportMsg` on the filter", func() {
				checkForFields(c, msg)
			})

			c.Specify("adds the channel data", func() {
				c.Expect(hasChannelData(msg), gs.IsTrue)
			})

			c.Specify("has its leak count set properly", func() {
				leakVal, ok := msg.GetFieldValue("LeakCount")
				c.Assume(ok, gs.IsTrue)
				i, ok := leakVal.(int64)
				c.Assume(ok, gs.IsTrue)
				c.Expect(int(i), gs.Equals, leakCount)
			})
		})

		c.Specify("w/ an input", func() {
			err := PopulateReportMsg(iRunner, msg)
			c.Assume(err, gs.IsNil)

			c.Specify("invokes `ReportMsg` on the input", func() {
				checkForFields(c, msg)
			})

			c.Specify("doesn't add any channel data", func() {
				capVal, ok := msg.GetFieldValue("InChanCapacity")
				c.Expect(capVal, gs.IsNil)
				c.Expect(ok, gs.IsFalse)
				lenVal, ok := msg.GetFieldValue("InChanLength")
				c.Expect(lenVal, gs.IsNil)
				c.Expect(ok, gs.IsFalse)
			})
		})
	})

	c.Specify("PipelineConfig", func() {
		pc := NewPipelineConfig(nil)
		// Initialize all of the PipelinePacks that we'll need
		pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)

		pc.FilterRunners = map[string]FilterRunner{fName: fRunner}
		pc.InputRunners = map[string]InputRunner{iName: iRunner}

		c.Specify("returns full set of accurate reports", func() {
			reportChan := make(chan *PipelinePack)
			go pc.reports(reportChan)

			reports := make(map[string]*PipelinePack)
			for r := range reportChan {
				iName, ok := r.Message.GetFieldValue("name")
				c.Expect(ok, gs.IsTrue)
				name, ok := iName.(string)
				c.Expect(ok, gs.IsTrue)
				c.Expect(name, gs.Not(gs.Equals), "MISSING")
				reports[name] = r
				pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)
			}
			fReport := reports[fName]
			c.Expect(fReport, gs.Not(gs.IsNil))
			checkForFields(c, fReport.Message)
			c.Expect(hasChannelData(fReport.Message), gs.IsTrue)

			iReport := reports[iName]
			c.Expect(iReport, gs.Not(gs.IsNil))
			checkForFields(c, iReport.Message)

			recycleReport := reports["inputRecycleChan"]
			c.Expect(recycleReport, gs.Not(gs.IsNil))
			capVal, ok := recycleReport.Message.GetFieldValue("InChanCapacity")
			c.Expect(ok, gs.IsTrue)
			c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))

			injectReport := reports["injectRecycleChan"]
			c.Expect(injectReport, gs.Not(gs.IsNil))
			capVal, ok = injectReport.Message.GetFieldValue("InChanCapacity")
			c.Expect(ok, gs.IsTrue)
			c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))

			routerReport := reports["Router"]
			c.Expect(routerReport, gs.Not(gs.IsNil))
			c.Expect(hasChannelData(routerReport.Message), gs.IsTrue)
		})
	})
}
Example #19
0
func StatAccumInputSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(InputTestHelper)
	ith.MockHelper = NewMockPluginHelper(ctrl)
	ith.MockInputRunner = NewMockInputRunner(ctrl)
	ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.PackSupply <- ith.Pack

	c.Specify("A StatAccumInput using normal namespaces", func() {
		ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Inject(ith.Pack)
		ith.MockInputRunner.EXPECT().Ticker()

		statAccumInput := StatAccumInput{}
		config := statAccumInput.ConfigStruct().(*StatAccumInputConfig)
		config.EmitInFields = true
		config.EmitInPayload = false
		err := statAccumInput.Init(config)
		c.Expect(err, gs.IsNil)

		var wg sync.WaitGroup
		prepareSendingStats := func() {
			wg.Add(1)
			go func() {
				err := statAccumInput.Run(ith.MockInputRunner, ith.MockHelper)
				wg.Done()
				c.Expect(err, gs.IsNil)
			}()
		}
		finalizeSendingStats := func() *message.Message {
			close(statAccumInput.statChan)
			wg.Wait()
			return ith.Pack.Message
		}

		sendTimer := func(key string, vals ...int) {
			for _, v := range vals {
				statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "ms", float32(1)}
			}
		}
		sendCounter := func(key string, vals ...int) {
			for _, v := range vals {
				statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "c", float32(1)}
			}
		}
		sendGauge := func(key string, vals ...int) {
			for _, v := range vals {
				statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "g", float32(1)}
			}
		}

		validateValueAtKey := func(msg *message.Message, key string, value interface{}) {
			fieldValue, ok := msg.GetFieldValue(key)
			c.Expect(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, value)
		}
		c.Specify("emits timer with correct prefixes", func() {
			prepareSendingStats()
			sendTimer("sample.timer", 10, 10, 20, 20)
			sendTimer("sample2.timer", 10, 20)
			msg := finalizeSendingStats()

			validateValueAtKey(msg, "sample.timer.count", int64(4))
			validateValueAtKey(msg, "sample.timer.mean", 15.0)
			validateValueAtKey(msg, "sample.timer.lower", 10.0)
			validateValueAtKey(msg, "sample2.timer.count", int64(2))
			validateValueAtKey(msg, "sample2.timer.mean", 15.0)
			validateValueAtKey(msg, "sample2.timer.lower", 10.0)

			validateValueAtKey(msg, "statsd.numStats", int64(2))
		})
		c.Specify("emits counters with correct prefixes", func() {
			prepareSendingStats()
			sendCounter("sample.cnt", 1, 2, 3, 4, 5)
			sendCounter("sample2.cnt", 159, 951)
			msg := finalizeSendingStats()

			validateValueAtKey(msg, "sample.cnt.count", int64(15))
			validateValueAtKey(msg, "sample.cnt.rate", 1.5)

			validateValueAtKey(msg, "sample2.cnt.count", int64(1110))
			validateValueAtKey(msg, "sample2.cnt.rate", 1110.0/float64(config.TickerInterval))
		})
		c.Specify("emits gauge with correct prefixes", func() {
			prepareSendingStats()
			sendGauge("sample.gauge", 1, 2)
			sendGauge("sample2.gauge", 1, 2, 3, 4, 5)
			msg := finalizeSendingStats()
			validateValueAtKey(msg, "sample.gauge", int64(2))
			validateValueAtKey(msg, "sample2.gauge", int64(5))
		})

		c.Specify("emits correct statsd.numStats count", func() {
			prepareSendingStats()
			sendGauge("sample.gauge", 1, 2)
			sendGauge("sample2.gauge", 1, 2)
			sendCounter("sample.cnt", 1, 2, 3, 4, 5)
			sendCounter("sample2.cnt", 159, 951)
			sendTimer("sample.timer", 10, 10, 20, 20)
			sendTimer("sample2.timer", 10, 20)
			msg := finalizeSendingStats()
			validateValueAtKey(msg, "statsd.numStats", int64(6))
		})
	})

	c.Specify("A StatAccumInput using Legacy namespaces", func() {
		statAccumInput := StatAccumInput{}
		config := statAccumInput.ConfigStruct().(*StatAccumInputConfig)
		config.LegacyNamespaces = true

		tickChan := make(chan time.Time)

		c.Specify("must emit data in payload and/or message fields", func() {
			config.EmitInPayload = false
			err := statAccumInput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
			expected := "One of either `EmitInPayload` or `EmitInFields` must be set to true."
			c.Expect(err.Error(), gs.Equals, expected)
		})

		c.Specify("that actually emits a message", func() {
			statName := "sample.stat"
			statVal := int64(303)
			testStat := Stat{statName, strconv.Itoa(int(statVal)), "c", float32(1)}

			validateMsgFields := func(msg *message.Message) {
				c.Expect(len(msg.Fields), gs.Equals, 4)

				// timestamp
				_, ok := msg.GetFieldValue("timestamp")
				c.Expect(ok, gs.IsTrue)

				var tmp interface{}
				var intTmp int64

				// stats.sample.stat
				tmp, ok = msg.GetFieldValue("stats." + statName)
				c.Expect(ok, gs.IsTrue)
				intTmp, ok = tmp.(int64)
				c.Expect(ok, gs.IsTrue)
				c.Expect(intTmp, gs.Equals, int64(30))

				// stats_counts.sample.stat
				tmp, ok = msg.GetFieldValue("stats_counts." + statName)
				c.Expect(ok, gs.IsTrue)
				intTmp, ok = tmp.(int64)
				c.Expect(ok, gs.IsTrue)
				c.Expect(intTmp, gs.Equals, statVal)

				// statsd.numStats
				tmp, ok = msg.GetFieldValue("statsd.numStats")

				c.Expect(ok, gs.IsTrue)
				intTmp, ok = tmp.(int64)
				c.Expect(ok, gs.IsTrue)
				c.Expect(intTmp, gs.Equals, int64(1))
			}

			validateMsgPayload := func(msg *message.Message) {
				lines := strings.Split(msg.GetPayload(), "\n")
				c.Expect(len(lines), gs.Equals, 4)
				c.Expect(lines[3], gs.Equals, "")

				var timestamp string
				for i := 0; i < 3; i++ {
					line := strings.Split(lines[i], " ")
					c.Expect(len(line), gs.Equals, 3)
					switch i {
					case 0:
						c.Expect(line[0], gs.Equals, "stats."+statName)
						c.Expect(line[1], gs.Equals, "30.300000")
						timestamp = line[2]
					case 1:
						c.Expect(line[0], gs.Equals, "stats_counts."+statName)
						c.Expect(line[1], gs.Equals, strconv.Itoa(int(statVal)))
						c.Expect(line[2], gs.Equals, timestamp)
					case 2:
						c.Expect(line[0], gs.Equals, "statsd.numStats")
						c.Expect(line[1], gs.Equals, "1")
						c.Expect(line[2], gs.Equals, timestamp)
					}
				}
				expected := strings.Join(lines, "\n")
				c.Expect(msg.GetPayload(), gs.Equals, expected)

			}

			ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
			ith.MockInputRunner.EXPECT().Inject(ith.Pack)
			ith.MockInputRunner.EXPECT().Ticker()

			var wg sync.WaitGroup

			startAndSwapTickChan := func() {
				wg.Add(1)
				go func() {
					err := statAccumInput.Run(ith.MockInputRunner, ith.MockHelper)
					wg.Done()
					c.Expect(err, gs.IsNil)
				}()
				time.Sleep(50) // Kludgey wait for tickChan to be set so we can replace.
				statAccumInput.tickChan = tickChan
			}

			c.Specify("emits data in payload by default", func() {
				err := statAccumInput.Init(config)
				c.Assume(err, gs.IsNil)

				startAndSwapTickChan()
				statAccumInput.statChan <- testStat
				close(statAccumInput.statChan)
				wg.Wait()

				validateMsgPayload(ith.Pack.Message)
			})

			c.Specify("emits data in fields when specified", func() {
				config.EmitInFields = true
				err := statAccumInput.Init(config)
				c.Assume(err, gs.IsNil)

				startAndSwapTickChan()
				statAccumInput.statChan <- testStat
				close(statAccumInput.statChan)
				wg.Wait()
				validateMsgFields(ith.Pack.Message)
				validateMsgPayload(ith.Pack.Message)
			})

			c.Specify("omits data in payload when specified", func() {
				config.EmitInPayload = false
				config.EmitInFields = true
				err := statAccumInput.Init(config)
				c.Assume(err, gs.IsNil)

				startAndSwapTickChan()
				statAccumInput.statChan <- testStat
				close(statAccumInput.statChan)
				wg.Wait()

				validateMsgFields(ith.Pack.Message)
				c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "")
			})

			c.Specify("honors time ticker to flush", func() {
				err := statAccumInput.Init(config)
				c.Assume(err, gs.IsNil)

				startAndSwapTickChan()
				statAccumInput.statChan <- testStat
				tickChan <- time.Now()
				validateMsgPayload(ith.Pack.Message)

				ith.PackSupply <- ith.Pack
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
				ith.MockInputRunner.EXPECT().Inject(ith.Pack)
				close(statAccumInput.statChan)
				wg.Wait()
			})

			c.Specify("correctly processes timers", func() {
				sendTimer := func(vals ...int) {
					for _, v := range vals {
						statAccumInput.statChan <- Stat{"sample.timer", strconv.Itoa(int(v)), "ms", float32(1)}
					}
				}
				config.EmitInFields = true
				err := statAccumInput.Init(config)
				c.Assume(err, gs.IsNil)
				startAndSwapTickChan()

				sendTimer(220, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100)

				close(statAccumInput.statChan)
				wg.Wait()

				msg := ith.Pack.Message

				getVal := func(token string) float64 {
					tmp, ok := msg.GetFieldValue("stats.timers.sample.timer." + token)
					c.Expect(ok, gs.IsTrue)
					val, ok := tmp.(float64)
					c.Expect(ok, gs.IsTrue)
					return val
				}

				c.Expect(getVal("upper"), gs.Equals, 220.0)
				c.Expect(getVal("lower"), gs.Equals, 10.0)
				c.Expect(getVal("mean"), gs.Equals, 70.0)
				c.Expect(getVal("upper_90"), gs.Equals, 100.0)
				c.Expect(getVal("mean_90"), gs.Equals, 55.0)
				tmp, ok := msg.GetFieldValue("stats.timers.sample.timer.count")
				c.Expect(ok, gs.IsTrue)
				intTmp, ok := tmp.(int64)
				c.Expect(ok, gs.IsTrue)
				c.Expect(intTmp, gs.Equals, int64(11))
			})
		})
	})
}
Example #20
0
func TcpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)
	key := "testkey"
	signers := map[string]Signer{"test_1": {key}}
	signer := "test"

	c.Specify("A TcpInput protobuf parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
		tcpInput := TcpInput{}
		err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
			Signers:    signers,
			Decoder:    "ProtobufDecoder",
			ParserType: "message.proto"})
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr)
		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		enccall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "TcpInput-123-ProtobufDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		cleanup := func() {
			mockListener.EXPECT().Close()
			tcpInput.Stop()
			tcpInput.wg.Wait()
		}

		c.Specify("reads a message from its connection", func() {
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
		})

		c.Specify("reads a MD5 signed message from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(md5.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-ith.DecodeChan:
				c.Expect(ith.Pack, gs.Equals, packRef)
				c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
				c.Expect(ith.Pack.Signer, gs.Equals, "test")
			case t := <-timeout:
				c.Expect(t, gs.IsNil)
			}
		})

		c.Specify("reads a SHA1 signed message from its connection", func() {
			header.SetHmacHashFunction(message.Header_SHA1)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(sha1.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-ith.DecodeChan:
				c.Expect(ith.Pack, gs.Equals, packRef)
				c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
				c.Expect(ith.Pack.Signer, gs.Equals, "test")
			case t := <-timeout:
				c.Expect(t, gs.IsNil)
			}
		})

		c.Specify("reads a signed message with an expired key from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(11)) // non-existent key version
			hm := hmac.New(md5.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-mockDecoderRunner.InChan():
				c.Expect(packRef, gs.IsNil)
			case t := <-timeout:
				c.Expect(t, gs.IsTrue)
			}
		})

		c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(md5.New, []byte(key))
			hm.Write([]byte("some bytes"))
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-mockDecoderRunner.InChan():
				c.Expect(packRef, gs.IsNil)
			case t := <-timeout:
				c.Expect(t, gs.IsTrue)
			}
		})
	})

	c.Specify("A TcpInput regexp parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")

		config := &TcpInputConfig{
			Net:        "tcp",
			Address:    ith.AddrStr,
			Decoder:    "RegexpDecoder",
			ParserType: "regexp",
		}

		tcpInput := TcpInput{}
		err := tcpInput.Init(config)
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
		mbytes := []byte("this is a test message\n")
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Name().Return("logger")
		enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "TcpInput-123-RegexpDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		c.Specify("reads a message from its connection", func() {
			readCall.Return(len(mbytes), nil)
			readCall.Do(getPayloadText(mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				mockListener.EXPECT().Close()
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes[:len(mbytes)-1]))
			c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
			c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
		})
	})

	c.Specify("A TcpInput token parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
		tcpInput := TcpInput{}
		err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
			Decoder:    "TokenDecoder",
			ParserType: "token",
			Delimiter:  "\n"})
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
		mbytes := []byte("this is a test message\n")
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Name().Return("logger")
		enccall := ith.MockHelper.EXPECT().DecoderRunner("TokenDecoder", "TcpInput-123-TokenDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		c.Specify("reads a message from its connection", func() {
			readCall.Return(len(mbytes), nil)
			readCall.Do(getPayloadText(mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				mockListener.EXPECT().Close()
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes))
			c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
			c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
		})
	})

	c.Specify("A TcpInput using TLS", func() {
		tcpInput := TcpInput{}
		config := &TcpInputConfig{
			Net:        "tcp",
			Address:    ith.AddrStr,
			ParserType: "token",
			UseTls:     true,
		}

		c.Specify("fails to init w/ missing key or cert file", func() {
			config.Tls = TlsConfig{}
			err := tcpInput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("accepts TLS client connections", func() {
			ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
			config.Tls = TlsConfig{
				CertFile: "./testsupport/cert.pem",
				KeyFile:  "./testsupport/key.pem",
			}
			err := tcpInput.Init(config)
			c.Expect(err, gs.IsNil)
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()

			clientConfig := new(tls.Config)
			clientConfig.InsecureSkipVerify = true
			conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
			c.Expect(err, gs.IsNil)
			defer conn.Close()
			conn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
			n, err := conn.Write([]byte("This is a test."))
			c.Expect(err, gs.IsNil)
			c.Expect(n, gs.Equals, len("This is a test."))
		})

		c.Specify("doesn't accept connections below specified min TLS version", func() {
			ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
			config.Tls = TlsConfig{
				CertFile:   "./testsupport/cert.pem",
				KeyFile:    "./testsupport/key.pem",
				MinVersion: "TLS12",
			}
			err := tcpInput.Init(config)
			c.Expect(err, gs.IsNil)
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				tcpInput.Stop()
				tcpInput.wg.Wait()
				time.Sleep(time.Duration(1000))
			}()

			clientConfig := &tls.Config{
				InsecureSkipVerify: true,
				MaxVersion:         tls.VersionTLS11,
			}
			conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
			c.Expect(conn, gs.IsNil)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
func ProcessDirectoryInputSpec(c gs.Context) {

	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	ith.PackSupply <- ith.Pack

	err := pConfig.RegisterDefault("NullSplitter")
	c.Assume(err, gs.IsNil)

	c.Specify("A ProcessDirectoryInput", func() {
		pdiInput := ProcessDirectoryInput{}
		pdiInput.SetPipelineConfig(pConfig)

		config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig)
		workingDir, err := os.Getwd()
		c.Assume(err, gs.IsNil)
		config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes")

		// `Ticker` is the last thing called during the setup part of the
		// input's `Run` method, so it triggers a waitgroup that tests can
		// wait on when they need to ensure initialization has finished.
		var started sync.WaitGroup
		started.Add(1)
		tickChan := make(chan time.Time, 1)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do(
			func() {
				started.Done()
			})

		// Similarly we use a waitgroup to signal when LogMessage has been
		// called to know when reloads have completed. Warning: If you call
		// expectLogMessage with a msg that is never passed to LogMessage and
		// then you call loaded.Wait() then your test will hang and never
		// complete.
		var loaded sync.WaitGroup
		expectLogMessage := func(msg string) {
			loaded.Add(1)
			ith.MockInputRunner.EXPECT().LogMessage(msg).Do(
				func(msg string) {
					loaded.Done()
				})
		}

		// Same name => same content.
		paths := []string{
			filepath.Join(config.ProcessDir, "100", "h0.toml"),
			filepath.Join(config.ProcessDir, "100", "h1.toml"),
			filepath.Join(config.ProcessDir, "200", "h0.toml"),
			filepath.Join(config.ProcessDir, "300", "h1.toml"),
		}

		copyFile := func(src, dest string) {
			inFile, err := os.Open(src)
			c.Assume(err, gs.IsNil)
			outFile, err := os.Create(dest)
			c.Assume(err, gs.IsNil)
			_, err = io.Copy(outFile, inFile)
			c.Assume(err, gs.IsNil)
			inFile.Close()
			outFile.Close()
		}

		err = pdiInput.Init(config)
		c.Expect(err, gs.IsNil)

		for _, p := range paths {
			expectLogMessage("Added: " + p)
		}
		go pdiInput.Run(ith.MockInputRunner, ith.MockHelper)
		defer func() {
			pdiInput.Stop()
			for _, entry := range pdiInput.inputs {
				entry.ir.Input().Stop()
			}
		}()
		started.Wait()

		c.Specify("loads scheduled jobs", func() {
			pathIndex := func(name string) (i int) {
				var p string
				for i, p = range paths {
					if name == p {
						return
					}
				}
				return -1
			}

			for name, entry := range pdiInput.inputs {
				i := pathIndex(name)
				// Make sure each file path got registered.
				c.Expect(i, gs.Not(gs.Equals), -1)
				dirName := filepath.Base(filepath.Dir(name))
				dirInt, err := strconv.Atoi(dirName)
				c.Expect(err, gs.IsNil)
				// And that the ticker interval was read correctly.
				c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval)
			}
		})

		c.Specify("discovers and adds a new job", func() {
			// Copy one of the files to register a new process.
			newPath := filepath.Join(config.ProcessDir, "300", "h0.toml")
			copyFile(paths[0], newPath)
			defer func() {
				err := os.Remove(newPath)
				c.Assume(err, gs.IsNil)
			}()

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Added: " + newPath)
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure our plugin was loaded.
			c.Expect(len(pdiInput.inputs), gs.Equals, 5)
			newEntry, ok := pdiInput.inputs[newPath]
			c.Expect(ok, gs.IsTrue)
			c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300))
		})

		c.Specify("removes a deleted job", func() {
			err := os.Remove(paths[3])
			c.Assume(err, gs.IsNil)
			defer func() {
				copyFile(paths[1], paths[3])
			}()

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Removed: " + paths[3])
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure our plugin was deleted.
			c.Expect(len(pdiInput.inputs), gs.Equals, 3)
		})

		c.Specify("notices a changed job", func() {
			// Overwrite one job w/ a slightly different one.
			copyFile(paths[0], paths[3])
			defer copyFile(paths[1], paths[3])

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Removed: " + paths[3])
			expectLogMessage("Added: " + paths[3])
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure the new config was loaded.
			c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals,
				"hello world\n")
		})
	})

}
Example #22
0
func ProcessChainSpec(c gs.Context) {

	readCommandOutput := func(reader io.Reader, resultChan chan string) {
		data, err := ioutil.ReadAll(reader)
		c.Expect(err, gs.IsNil)
		resultChan <- string(data)
	}

	c.Specify("A ManagedCommand", func() {

		c.Specify("can run a single command", func() {
			Path := SINGLE_CMD
			cmd := NewManagedCmd(Path, SINGLE_CMD_ARGS, 0)
			output := make(chan string)
			cmd.Start(true)
			go readCommandOutput(cmd.Stdout_r, output)
			cmd.Wait()
			outputStr := <-output
			c.Expect(fmt.Sprintf("%x", outputStr), gs.Equals,
				fmt.Sprintf("%x", SINGLE_CMD_OUTPUT))
		})

		c.Specify("honors nonzero timeouts", func() {
			// Timeout should always occur inside of 10 seconds.
			Path := NONZERO_TIMEOUT_CMD
			timeout := NONZERO_TIMEOUT
			cmd := NewManagedCmd(Path, NONZERO_TIMEOUT_ARGS, timeout)
			output := make(chan string)
			cmd.Start(true)
			start := time.Now()
			go readCommandOutput(cmd.Stdout_r, output)
			cmd.Wait()
			end := time.Now()
			<-output
			actualDuration := end.Sub(start)
			c.Expect(actualDuration < time.Second*10, gs.Equals, true)
		})

		c.Specify("reads process stderr properly", func() {
			Path := STDERR_CMD
			cmd := NewManagedCmd(Path, STDERR_CMD_ARGS, 0)

			stdoutResults := make(chan string, 1)
			stderrResults := make(chan string, 1)

			cmd.Start(true)
			go readCommandOutput(cmd.Stdout_r, stdoutResults)
			go readCommandOutput(cmd.Stderr_r, stderrResults)
			cmd.Wait()

			// Error messages will vary platform to platform, just check that
			// there is some message which will be about "No such file found".
			c.Expect(len(<-stderrResults) > 0, gs.Equals, true)
			c.Expect(<-stdoutResults, gs.Equals, "")
		})

		c.Specify("can be terminated before timeout occurs", func() {
			Path := NONZERO_TIMEOUT_CMD
			timeout := time.Second * 30
			cmd := NewManagedCmd(Path, NONZERO_TIMEOUT_ARGS, timeout)

			stdoutResults := make(chan string, 1)

			cmd.Start(true)
			start := time.Now()
			go readCommandOutput(cmd.Stdout_r, stdoutResults)
			time.Sleep(NONZERO_TIMEOUT)
			cmd.Stopchan <- true
			cmd.Wait()
			end := time.Now()

			actualDuration := end.Sub(start)
			c.Expect(actualDuration < timeout, gs.Equals, true)
		})

		c.Specify("can reset commands to run again", func() {
			Path := SINGLE_CMD
			cmd := NewManagedCmd(Path, SINGLE_CMD_ARGS, 0)

			stdoutResults := make(chan string, 1)

			cmd.Start(true)
			go readCommandOutput(cmd.Stdout_r, stdoutResults)
			cmd.Wait()
			c.Expect(<-stdoutResults, gs.Equals, SINGLE_CMD_OUTPUT)

			// Reset and rerun it.
			cmd = cmd.clone()
			cmd.Start(true)
			go readCommandOutput(cmd.Stdout_r, stdoutResults)
			cmd.Wait()
			c.Expect(<-stdoutResults, gs.Equals, SINGLE_CMD_OUTPUT)
		})
	})

	c.Specify("A New ProcessChain", func() {
		c.Specify("pipes multiple commands correctly", func() {
			chain := NewCommandChain(0)
			chain.AddStep(PIPE_CMD1, PIPE_CMD1_ARGS...)
			chain.AddStep(PIPE_CMD2, PIPE_CMD2_ARGS...)

			err := chain.Start()
			c.Expect(err, gs.IsNil)

			stdoutReader, err := chain.Stdout_r()
			c.Expect(err, gs.IsNil)
			stdoutResult := make(chan string, 1)

			stderrReader, err := chain.Stderr_r()
			c.Expect(err, gs.IsNil)
			stderrResult := make(chan string, 1)

			go readCommandOutput(stdoutReader, stdoutResult)
			go readCommandOutput(stderrReader, stderrResult)

			err = chain.Wait()
			c.Expect(err, gs.IsNil)

			c.Expect(<-stderrResult, gs.Equals, "")
			c.Expect(<-stdoutResult, gs.Equals, PIPE_CMD_OUTPUT)

		})

		c.Specify("will honor timeouts", func() {
			// This test must timeout within 10 seconds.
			var err error

			timeout := NONZERO_TIMEOUT
			chain := NewCommandChain(timeout)
			chain.AddStep(TIMEOUT_PIPE_CMD1, TIMEOUT_PIPE_CMD1_ARGS...)
			chain.AddStep(TIMEOUT_PIPE_CMD2, TIMEOUT_PIPE_CMD2_ARGS...)

			err = chain.Start()
			start := time.Now()
			c.Expect(err, gs.IsNil)
			err = chain.Wait()
			end := time.Now()
			actual_duration := end.Sub(start)
			c.Expect(err, gs.Not(gs.IsNil))
			c.Expect(strings.Contains(err.Error(), "was killed"), gs.Equals, true)
			c.Expect(actual_duration < time.Second*10, gs.Equals, true)
		})

		c.Specify("will stop chains before timeout has completed", func() {
			var err error

			timeout := time.Second * 30
			chain := NewCommandChain(timeout)
			chain.AddStep(TIMEOUT_PIPE_CMD1, TIMEOUT_PIPE_CMD1_ARGS...)
			chain.AddStep(TIMEOUT_PIPE_CMD2, TIMEOUT_PIPE_CMD2_ARGS...)

			err = chain.Start()
			start := time.Now()
			c.Expect(err, gs.IsNil)
			time.Sleep(NONZERO_TIMEOUT)

			chain.Stopchan <- true
			err = chain.Wait()
			end := time.Now()
			actual_duration := end.Sub(start)
			c.Expect(err, gs.Not(gs.IsNil))
			c.Expect(actual_duration < timeout, gs.Equals, true)
		})

		c.Specify("can reset chains to run again", func() {
			// This test assumes tail and grep
			var err error

			chain := NewCommandChain(0)
			chain.AddStep(PIPE_CMD1, PIPE_CMD1_ARGS...)
			chain.AddStep(PIPE_CMD2, PIPE_CMD2_ARGS...)

			err = chain.Start()
			c.Expect(err, gs.IsNil)

			stdoutReader, err := chain.Stdout_r()
			c.Expect(err, gs.IsNil)
			stdoutResult := make(chan string, 1)

			stderrReader, err := chain.Stderr_r()
			c.Expect(err, gs.IsNil)
			stderrResult := make(chan string, 1)

			go readCommandOutput(stdoutReader, stdoutResult)
			go readCommandOutput(stderrReader, stderrResult)

			err = chain.Wait()
			c.Expect(err, gs.IsNil)

			c.Expect(<-stderrResult, gs.Equals, "")
			c.Expect(<-stdoutResult, gs.Equals, PIPE_CMD_OUTPUT)

			// Reset the chain for a second run
			chain = chain.clone()

			err = chain.Start()
			c.Expect(err, gs.IsNil)

			stdoutReader, err = chain.Stdout_r()
			c.Expect(err, gs.IsNil)

			stderrReader, err = chain.Stderr_r()
			c.Expect(err, gs.IsNil)

			go readCommandOutput(stdoutReader, stdoutResult)
			go readCommandOutput(stderrReader, stderrResult)

			err = chain.Wait()
			c.Expect(err, gs.IsNil)

			c.Expect(<-stderrResult, gs.Equals, "")
			c.Expect(<-stdoutResult, gs.Equals, PIPE_CMD_OUTPUT)
		})
	})
}
Example #23
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	config := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: config}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}
		lsiConfig.Decoder = "decoder-name"

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)
			mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
				// Expect calls to get decoder and decode each message. Since the
				// decoding is a no-op, the message payload will be the log file
				// line, unchanged.
				pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
					"-"+lsiConfig.Decoder)
				pbcall.Return(mockDecoderRunner, true)

				decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
				decodeCall.Return(ith.DecodeChan)

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				d, _ := time.ParseDuration("5s")
				timeout := time.After(d)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case <-ith.DecodeChan:
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Example #24
0
func ESEncodersSpec(c gs.Context) {
	recycleChan := make(chan *PipelinePack, 1)
	pack := NewPipelinePack(recycleChan)
	pack.Message = getTestMessageWithFunnyFields()

	c.Specify("writeStringField", func() {
		buf := bytes.Buffer{}
		c.Specify("should properly encode special characters in json", func() {
			writeStringField(true, &buf, `hello"bar`, "world\nfoo\\")
			c.Expect(buf.String(), gs.Equals, `"hello\u0022bar":"world\u000afoo\u005c"`)
		})
		c.Specify("Should replace invalid utf8 with replacement character", func() {
			writeStringField(true, &buf, "\xa3", "\xa3")
			c.Expect(buf.String(), gs.Equals, "\"\xEF\xBF\xBD\":\"\xEF\xBF\xBD\"")
		})
	})

	c.Specify("interpolateFlag", func() {
		c.Specify("should interpolate for index and type names", func() {
			interpolatedIndex, err := interpolateFlag(&ElasticSearchCoordinates{},
				pack.Message, "heka-%{Pid}-%{\"foo}-%{%Y.%m.%d}")
			c.Expect(err, gs.IsNil)
			t := time.Now().UTC()
			c.Expect(interpolatedIndex, gs.Equals, "heka-14098-bar\n-"+t.Format("2006.01.02"))

			interpolatedType, err := interpolateFlag(&ElasticSearchCoordinates{},
				pack.Message, "%{Type}")
			c.Expect(err, gs.IsNil)
			c.Expect(interpolatedType, gs.Equals, "TEST")
		})

		c.Specify("should interpolate from message field", func() {
			id := "%{idField}"
			interpolatedId, err := interpolateFlag(&ElasticSearchCoordinates{},
				pack.Message, id)
			c.Expect(err, gs.IsNil)
			c.Expect(interpolatedId, gs.Equals, "1234")
		})

		c.Specify("should fail for nonexistent message field", func() {
			id := "%{idFail}"
			unInterpolatedId, err := interpolateFlag(&ElasticSearchCoordinates{},
				pack.Message, id)

			c.Expect(strings.Contains(err.Error(),
				"Could not interpolate field from config: %{idFail}"), gs.IsTrue)
			c.Expect(unInterpolatedId, gs.Equals, "idFail")
		})
	})

	c.Specify("ESLogstashV0Encoder", func() {
		encoder := new(ESLogstashV0Encoder)
		config := encoder.ConfigStruct().(*ESLogstashV0EncoderConfig)
		config.RawBytesFields = []string{
			"test_raw_field_string",
			"test_raw_field_bytes",
			"test_raw_field_string_array",
			"test_raw_field_bytes_array",
		}

		c.Specify("Should properly encode a message", func() {
			err := encoder.Init(config)
			c.Assume(err, gs.IsNil)
			b, err := encoder.Encode(pack)
			c.Assume(err, gs.IsNil)

			output := string(b)
			lines := strings.Split(output, string(NEWLINE))

			decoded := make(map[string]interface{})
			err = json.Unmarshal([]byte(lines[0]), &decoded)
			c.Assume(err, gs.IsNil)
			sub := decoded["index"].(map[string]interface{})
			t := time.Now().UTC()
			c.Expect(sub["_index"], gs.Equals, "logstash-"+t.Format("2006.01.02"))
			c.Expect(sub["_type"], gs.Equals, "message")

			decoded = make(map[string]interface{})
			err = json.Unmarshal([]byte(lines[1]), &decoded)
			c.Assume(err, gs.IsNil)
			c.Expect(decoded["@fields"].(map[string]interface{})[`"foo`], gs.Equals, "bar\n")
			c.Expect(decoded["@fields"].(map[string]interface{})[`"number`], gs.Equals, 64.0)
			c.Expect(decoded["@fields"].(map[string]interface{})["\xEF\xBF\xBD"], gs.Equals,
				"\xEF\xBF\xBD")
			c.Expect(decoded["@uuid"], gs.Equals, "87cf1ac2-e810-4ddf-a02d-a5ce44d13a85")
			c.Expect(decoded["@timestamp"], gs.Equals, "2013-07-16T15:49:05")
			c.Expect(decoded["@type"], gs.Equals, "message")
			c.Expect(decoded["@logger"], gs.Equals, "GoSpec")
			c.Expect(decoded["@severity"], gs.Equals, 6.0)
			c.Expect(decoded["@message"], gs.Equals, "Test Payload")
			c.Expect(decoded["@envversion"], gs.Equals, "0.8")
			c.Expect(decoded["@pid"], gs.Equals, 14098.0)
			c.Expect(decoded["@source_host"], gs.Equals, "hostname")

			stringsArray := decoded["@fields"].(map[string]interface{})["stringArray"].([]interface{})
			c.Expect(len(stringsArray), gs.Equals, 4)
			c.Expect(stringsArray[0], gs.Equals, "asdf")
			c.Expect(stringsArray[1], gs.Equals, "jkl;")
			c.Expect(stringsArray[2], gs.Equals, "push")
			c.Expect(stringsArray[3], gs.Equals, "pull")

			bytesArray := decoded["@fields"].(map[string]interface{})["byteArray"].([]interface{})
			c.Expect(len(bytesArray), gs.Equals, 2)
			c.Expect(bytesArray[0], gs.Equals, base64.StdEncoding.EncodeToString([]byte("asdf")))
			c.Expect(bytesArray[1], gs.Equals, base64.StdEncoding.EncodeToString([]byte("jkl;")))

			integerArray := decoded["@fields"].(map[string]interface{})["integerArray"].([]interface{})
			c.Expect(len(integerArray), gs.Equals, 3)
			c.Expect(integerArray[0], gs.Equals, 22.0)
			c.Expect(integerArray[1], gs.Equals, 80.0)
			c.Expect(integerArray[2], gs.Equals, 3000.0)

			doubleArray := decoded["@fields"].(map[string]interface{})["doubleArray"].([]interface{})
			c.Expect(len(doubleArray), gs.Equals, 2)
			c.Expect(doubleArray[0], gs.Equals, 42.0)
			c.Expect(doubleArray[1], gs.Equals, 19101.3)

			boolArray := decoded["@fields"].(map[string]interface{})["boolArray"].([]interface{})
			c.Expect(len(boolArray), gs.Equals, 5)
			c.Expect(boolArray[0], gs.Equals, true)
			c.Expect(boolArray[1], gs.Equals, false)
			c.Expect(boolArray[2], gs.Equals, false)
			c.Expect(boolArray[3], gs.Equals, false)
			c.Expect(boolArray[4], gs.Equals, true)

			c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_string"].(map[string]interface{})["asdf"], gs.Equals, 123.0)
			c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_bytes"].(map[string]interface{})["asdf"], gs.Equals, 123.0)
			c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_string_array"].([]interface{})[0].(map[string]interface{})["asdf"], gs.Equals, 123.0)
			c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_string_array"].([]interface{})[1].(map[string]interface{})["jkl;"], gs.Equals, 123.0)
			c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_bytes_array"].([]interface{})[0].(map[string]interface{})["asdf"], gs.Equals, 123.0)
			c.Expect(decoded["@fields"].(map[string]interface{})["test_raw_field_bytes_array"].([]interface{})[1].(map[string]interface{})["jkl;"], gs.Equals, 123.0)
		})

		c.Specify("encodes w/ a different timestamp format", func() {
			config.Timestamp = "%Y/%m/%d %H:%M:%S %z"
			err := encoder.Init(config)
			c.Expect(err, gs.IsNil)
			b, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)

			output := string(b)
			lines := strings.Split(output, string(NEWLINE))
			decoded := make(map[string]interface{})
			err = json.Unmarshal([]byte(lines[1]), &decoded)
			c.Assume(err, gs.IsNil)

			c.Expect(decoded["@timestamp"], gs.Equals, "2013/07/16 15:49:05 +0000")
		})

		c.Specify("validates dynamic_fields against fields list", func() {
			// "DynamicFields" not listed fails init.
			config.DynamicFields = []string{"asdf", "jkl;"}
			config.Fields = []string{"Logger", "Hostname"}
			err := encoder.Init(config)
			c.Assume(err, gs.Not(gs.IsNil))
			msg := "\"DynamicFields\" must be in 'fields' list if using 'dynamic_fields'"
			c.Expect(err.Error(), gs.Equals, msg)

			// "DynamicFields" listed passes init.
			config.Fields = []string{"Logger", "Hostname", "DynamicFields"}
			err = encoder.Init(config)
			c.Expect(err, gs.IsNil)
			c.Expect(len(encoder.dynamicFields), gs.Equals, 2)

			// "Fields" works as an alias for "DynamicFields".
			config.Fields = []string{"Logger", "Hostname", "Fields"}
			err = encoder.Init(config)
			c.Expect(err, gs.IsNil)
			c.Expect(len(encoder.dynamicFields), gs.Equals, 2)
		})

		c.Specify("honors dynamic fields", func() {
			c.Specify("when dynamic_fields is empty", func() {
				err := encoder.Init(config)
				c.Expect(err, gs.IsNil)
				b, err := encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				output := string(b)
				lines := strings.Split(output, string(NEWLINE))
				decoded := make(map[string]interface{})
				err = json.Unmarshal([]byte(lines[1]), &decoded)
				c.Assume(err, gs.IsNil)
				c.Expect(len(decoded), gs.Equals, 10)
				fieldsValInterface := decoded["@fields"]
				fieldsVal := fieldsValInterface.(map[string]interface{})
				c.Expect(len(fieldsVal), gs.Equals, 13)
			})
		})
	})

	c.Specify("ESJsonEncoder", func() {
		encoder := new(ESJsonEncoder)
		config := encoder.ConfigStruct().(*ESJsonEncoderConfig)
		config.RawBytesFields = []string{
			"test_raw_field_string",
			"test_raw_field_bytes",
			"test_raw_field_string_array",
			"test_raw_field_bytes_array",
		}

		c.Specify("Should properly encode a message", func() {
			err := encoder.Init(config)
			c.Assume(err, gs.IsNil)
			b, err := encoder.Encode(pack)
			c.Assume(err, gs.IsNil)

			output := string(b)
			lines := strings.Split(output, string(NEWLINE))

			decoded := make(map[string]interface{})
			err = json.Unmarshal([]byte(lines[0]), &decoded)
			c.Assume(err, gs.IsNil)
			sub := decoded["index"].(map[string]interface{})
			t := time.Now().UTC()
			c.Expect(sub["_index"], gs.Equals, "heka-"+t.Format("2006.01.02"))
			c.Expect(sub["_type"], gs.Equals, "message")

			decoded = make(map[string]interface{})
			err = json.Unmarshal([]byte(lines[1]), &decoded)
			c.Assume(err, gs.IsNil)
			c.Expect(decoded[`"foo`], gs.Equals, "bar\n")
			c.Expect(decoded[`"number`], gs.Equals, 64.0)
			c.Expect(decoded["\xEF\xBF\xBD"], gs.Equals, "\xEF\xBF\xBD")
			c.Expect(decoded["Uuid"], gs.Equals, "87cf1ac2-e810-4ddf-a02d-a5ce44d13a85")
			c.Expect(decoded["Timestamp"], gs.Equals, "2013-07-16T15:49:05")
			c.Expect(decoded["Type"], gs.Equals, "TEST")
			c.Expect(decoded["Logger"], gs.Equals, "GoSpec")
			c.Expect(decoded["Severity"], gs.Equals, 6.0)
			c.Expect(decoded["Payload"], gs.Equals, "Test Payload")
			c.Expect(decoded["EnvVersion"], gs.Equals, "0.8")
			c.Expect(decoded["Pid"], gs.Equals, 14098.0)
			c.Expect(decoded["Hostname"], gs.Equals, "hostname")

			stringsArray := decoded["stringArray"].([]interface{})
			c.Expect(len(stringsArray), gs.Equals, 4)
			c.Expect(stringsArray[0], gs.Equals, "asdf")
			c.Expect(stringsArray[1], gs.Equals, "jkl;")
			c.Expect(stringsArray[2], gs.Equals, "push")
			c.Expect(stringsArray[3], gs.Equals, "pull")

			bytesArray := decoded["byteArray"].([]interface{})
			c.Expect(len(bytesArray), gs.Equals, 2)
			c.Expect(bytesArray[0], gs.Equals, base64.StdEncoding.EncodeToString([]byte("asdf")))
			c.Expect(bytesArray[1], gs.Equals, base64.StdEncoding.EncodeToString([]byte("jkl;")))

			integerArray := decoded["integerArray"].([]interface{})
			c.Expect(len(integerArray), gs.Equals, 3)
			c.Expect(integerArray[0], gs.Equals, 22.0)
			c.Expect(integerArray[1], gs.Equals, 80.0)
			c.Expect(integerArray[2], gs.Equals, 3000.0)

			doubleArray := decoded["doubleArray"].([]interface{})
			c.Expect(len(doubleArray), gs.Equals, 2)
			c.Expect(doubleArray[0], gs.Equals, 42.0)
			c.Expect(doubleArray[1], gs.Equals, 19101.3)

			boolArray := decoded["boolArray"].([]interface{})
			c.Expect(len(boolArray), gs.Equals, 5)
			c.Expect(boolArray[0], gs.Equals, true)
			c.Expect(boolArray[1], gs.Equals, false)
			c.Expect(boolArray[2], gs.Equals, false)
			c.Expect(boolArray[3], gs.Equals, false)
			c.Expect(boolArray[4], gs.Equals, true)

			c.Expect(decoded["test_raw_field_string"].(map[string]interface{})["asdf"], gs.Equals, 123.0)
			c.Expect(decoded["test_raw_field_bytes"].(map[string]interface{})["asdf"], gs.Equals, 123.0)
			c.Expect(decoded["test_raw_field_string_array"].([]interface{})[0].(map[string]interface{})["asdf"], gs.Equals, 123.0)
			c.Expect(decoded["test_raw_field_string_array"].([]interface{})[1].(map[string]interface{})["jkl;"], gs.Equals, 123.0)
			c.Expect(decoded["test_raw_field_bytes_array"].([]interface{})[0].(map[string]interface{})["asdf"], gs.Equals, 123.0)
			c.Expect(decoded["test_raw_field_bytes_array"].([]interface{})[1].(map[string]interface{})["jkl;"], gs.Equals, 123.0)
		})

		c.Specify("Should use field mappings", func() {
			config := encoder.ConfigStruct().(*ESJsonEncoderConfig)
			config.FieldMappings = &ESFieldMappings{
				Timestamp:  "XTimestamp",
				Uuid:       "XUuid",
				Type:       "XType",
				Logger:     "XLogger",
				Severity:   "XSeverity",
				Payload:    "XPayload",
				EnvVersion: "XEnvVersion",
				Pid:        "XPid",
				Hostname:   "XHostname",
			}
			err := encoder.Init(config)
			c.Assume(err, gs.IsNil)
			b, err := encoder.Encode(pack)
			c.Assume(err, gs.IsNil)

			output := string(b)
			lines := strings.Split(output, string(NEWLINE))

			decoded := make(map[string]interface{})
			err = json.Unmarshal([]byte(lines[1]), &decoded)
			c.Assume(err, gs.IsNil)
			c.Expect(decoded["XTimestamp"], gs.Equals, "2013-07-16T15:49:05")
			c.Expect(decoded["XUuid"], gs.Equals, "87cf1ac2-e810-4ddf-a02d-a5ce44d13a85")
			c.Expect(decoded["XType"], gs.Equals, "TEST")
			c.Expect(decoded["XLogger"], gs.Equals, "GoSpec")
			c.Expect(decoded["XSeverity"], gs.Equals, 6.0)
			c.Expect(decoded["XPayload"], gs.Equals, "Test Payload")
			c.Expect(decoded["XEnvVersion"], gs.Equals, "0.8")
			c.Expect(decoded["XPid"], gs.Equals, 14098.0)
			c.Expect(decoded["XHostname"], gs.Equals, "hostname")
		})

		c.Specify("encodes w/ a different timestamp format", func() {
			config.Timestamp = "%Y/%m/%d %H:%M:%S %z"
			err := encoder.Init(config)
			c.Expect(err, gs.IsNil)
			b, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)

			output := string(b)
			lines := strings.Split(output, string(NEWLINE))
			decoded := make(map[string]interface{})
			err = json.Unmarshal([]byte(lines[1]), &decoded)
			c.Assume(err, gs.IsNil)

			c.Expect(decoded["Timestamp"], gs.Equals, "2013/07/16 15:49:05 +0000")
		})

		c.Specify("validates field list", func() {
			config.Fields = []string{"severity", "Hogger", "Lostname"}
			err := encoder.Init(config)
			c.Assume(err, gs.Not(gs.IsNil))
			msg := "Unsupported value \"Hogger\" in 'fields' list, must be one of"
			c.Expect(err.Error()[:len(msg)], gs.Equals, msg)
		})

		c.Specify("validates dynamic_fields against fields list", func() {
			// "DynamicFields" not listed fails init.
			config.DynamicFields = []string{"asdf", "jkl;"}
			config.Fields = []string{"Logger", "Hostname"}
			err := encoder.Init(config)
			c.Assume(err, gs.Not(gs.IsNil))
			msg := "\"DynamicFields\" must be in 'fields' list if using 'dynamic_fields'"
			c.Expect(err.Error(), gs.Equals, msg)

			// "DynamicFields" listed passes init.
			config.Fields = []string{"Logger", "Hostname", "DynamicFields"}
			err = encoder.Init(config)
			c.Expect(err, gs.IsNil)
			c.Expect(len(encoder.dynamicFields), gs.Equals, 2)

			// "Fields" works as an alias for "DynamicFields".
			config.Fields = []string{"Logger", "Hostname", "Fields"}
			err = encoder.Init(config)
			c.Expect(err, gs.IsNil)
			c.Expect(len(encoder.dynamicFields), gs.Equals, 2)
		})

		c.Specify("honors dynamic fields", func() {

			c.Specify("when dynamic_fields is empty", func() {
				err := encoder.Init(config)
				c.Assume(err, gs.IsNil)
				b, err := encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				output := string(b)
				lines := strings.Split(output, string(NEWLINE))
				decoded := make(map[string]interface{})
				err = json.Unmarshal([]byte(lines[1]), &decoded)
				c.Assume(err, gs.IsNil)
				c.Expect(len(decoded), gs.Equals, 22) // 9 base fields and 13 dynamic fields.
			})

			c.Specify("when dynamic_fields is specified", func() {
				config.DynamicFields = []string{"idField", "stringArray"}
				err := encoder.Init(config)
				c.Assume(err, gs.IsNil)
				b, err := encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				output := string(b)
				lines := strings.Split(output, string(NEWLINE))
				decoded := make(map[string]interface{})
				err = json.Unmarshal([]byte(lines[1]), &decoded)
				c.Assume(err, gs.IsNil)
				c.Expect(len(decoded), gs.Equals, 11) // 9 base fields and 2 dynamic fields.
			})
		})
	})
}
Example #25
0
func LoadFromConfigSpec(c gs.Context) {
	origGlobals := Globals

	origAvailablePlugins := make(map[string]func() interface{})
	for k, v := range AvailablePlugins {
		origAvailablePlugins[k] = v
	}

	pipeConfig := NewPipelineConfig(nil)
	defer func() {
		Globals = origGlobals
		AvailablePlugins = origAvailablePlugins
	}()

	c.Assume(pipeConfig, gs.Not(gs.IsNil))

	c.Specify("Config file loading", func() {
		c.Specify("works w/ good config file", func() {
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_test.toml")
			c.Assume(err, gs.IsNil)

			// We use a set of Expect's rather than c.Specify because the
			// pipeConfig can't be re-loaded per child as gospec will do
			// since each one needs to bind to the same address

			// and the inputs section loads properly with a custom name
			udp, ok := pipeConfig.InputRunners["UdpInput"]
			c.Expect(ok, gs.Equals, true)

			// and the decoders sections load
			_, ok = pipeConfig.DecoderWrappers["JsonDecoder"]
			c.Expect(ok, gs.Equals, false)
			_, ok = pipeConfig.DecoderWrappers["ProtobufDecoder"]
			c.Expect(ok, gs.Equals, true)

			// and the outputs section loads
			_, ok = pipeConfig.OutputRunners["LogOutput"]
			c.Expect(ok, gs.Equals, true)

			// and the filters sections loads
			_, ok = pipeConfig.FilterRunners["sample"]
			c.Expect(ok, gs.Equals, true)

			// Shut down UdpInput to free up the port for future tests.
			udp.Input().Stop()
		})

		c.Specify("works w/ decoder defaults", func() {
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_defaults.toml")
			c.Assume(err, gs.Not(gs.IsNil))

			// Only the ProtobufDecoder is loaded
			c.Expect(len(pipeConfig.DecoderWrappers), gs.Equals, 1)
		})

		c.Specify("works w/ MultiDecoder", func() {
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_multidecoder.toml")
			c.Assume(err, gs.IsNil)
			hasSyncDecoder := false

			// ProtobufDecoder will always be loaded
			c.Assume(len(pipeConfig.DecoderWrappers), gs.Equals, 2)

			// Check that the MultiDecoder actually loaded
			for k, _ := range pipeConfig.DecoderWrappers {
				if k == "syncdecoder" {
					hasSyncDecoder = true
					break
				}
			}
			c.Assume(hasSyncDecoder, gs.IsTrue)
		})

		c.Specify("explodes w/ bad config file", func() {
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_bad_test.toml")
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), ts.StringContains, "2 errors loading plugins")
			c.Expect(pipeConfig.LogMsgs, gs.ContainsAny, gs.Values("No such plugin: CounterOutput"))
		})

		c.Specify("handles missing config file correctly", func() {
			err := pipeConfig.LoadFromConfigFile("no_such_file.toml")
			c.Assume(err, gs.Not(gs.IsNil))
			if runtime.GOOS == "windows" {
				c.Expect(err.Error(), ts.StringContains, "open no_such_file.toml: The system cannot find the file specified.")
			} else {
				c.Expect(err.Error(), ts.StringContains, "open no_such_file.toml: no such file or directory")
			}
		})

		c.Specify("errors correctly w/ bad outputs config", func() {
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_bad_outputs.toml")
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), ts.StringContains, "1 errors loading plugins")
			msg := pipeConfig.LogMsgs[0]
			c.Expect(msg, ts.StringContains, "No such plugin")
		})

		c.Specify("for a DefaultsTestOutput", func() {
			RegisterPlugin("DefaultsTestOutput", func() interface{} {
				return new(DefaultsTestOutput)
			})
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_defaults2.toml")
			c.Expect(err, gs.IsNil)
			runner, ok := pipeConfig.OutputRunners["DefaultsTestOutput"]
			c.Expect(ok, gs.IsTrue)
			ticker := runner.Ticker()
			c.Expect(ticker, gs.Not(gs.IsNil))
			matcher := runner.MatchRunner().MatcherSpecification().String()
			c.Expect(matcher, gs.Equals, messageMatchStr)
		})

		c.Specify("can render JSON reports as pipe delimited data", func() {
			RegisterPlugin("DefaultsTestOutput", func() interface{} {
				return new(DefaultsTestOutput)
			})
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_defaults2.toml")
			c.Expect(err, gs.IsNil)

			data := `{"globals":[{"Name":"inputRecycleChan","InChanCapacity":{"value":"100", "representation":"count"},"InChanLength":{"value":"99", "representation":"count"}},{"Name":"injectRecycleChan","InChanCapacity":{"value":"100", "representation":"count"},"InChanLength":{"value":"98", "representation":"count"}},{"Name":"Router","InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"ProcessMessageCount":{"value":"26", "representation":"count"}}], "inputs": [{"Name": "TcpInput"}], "decoders": [{"Name":"ProtobufDecoder","InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"}}], "filters": [{"Name":"OpsSandboxManager","RunningFilters":{"value":"0", "representation":"count"},"ProcessMessageCount":{"value":"0", "representation":"count"},"InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"MatchChanCapacity":{"value":"50", "representation":"count"},"MatchChanLength":{"value":"0", "representation":"count"},"MatchAvgDuration":{"value":"0", "representation":"ns"}},{"Name":"hekabench_counter","Memory":{"value":"20644", "representation":"B"},"MaxMemory":{"value":"20644", "representation":"B"},"MaxInstructions":{"value":"18", "representation":"count"},"MaxOutput":{"value":"0", "representation":"B"},"ProcessMessageCount":{"value":"0", "representation":"count"},"InjectMessageCount":{"value":"0", "representation":"count"},"ProcessMessageAvgDuration":{"value":"0", "representation":"ns"},"TimerEventAvgDuration":{"value":"78532", "representation":"ns"},"InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"MatchChanCapacity":{"value":"50", "representation":"count"},"MatchChanLength":{"value":"0", "representation":"count"},"MatchAvgDuration":{"value":"445", "representation":"ns"}}], "outputs": [{"Name":"LogOutput","InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"MatchChanCapacity":{"value":"50", "representation":"count"},"MatchChanLength":{"value":"0", "representation":"count"},"MatchAvgDuration":{"value":"406", "representation":"ns"}},{"Name":"DashboardOutput","InChanCapacity":{"value":"50", "representation":"count"},"InChanLength":{"value":"0", "representation":"count"},"MatchChanCapacity":{"value":"50", "representation":"count"},"MatchChanLength":{"value":"0", "representation":"count"},"MatchAvgDuration":{"value":"336", "representation":"ns"}}]} `

			report := pipeConfig.FormatTextReport("heka.all-report", data)

			expected := `========[heka.all-report]========

====Globals====
inputRecycleChan:
    InChanCapacity: 100
    InChanLength: 99
injectRecycleChan:
    InChanCapacity: 100
    InChanLength: 98
Router:
    InChanCapacity: 50
    InChanLength: 0
    ProcessMessageCount: 26

====Inputs====
TcpInput:

====Decoders====
ProtobufDecoder:
    InChanCapacity: 50
    InChanLength: 0

====Filters====
OpsSandboxManager:
    InChanCapacity: 50
    InChanLength: 0
    MatchChanCapacity: 50
    MatchChanLength: 0
    MatchAvgDuration: 0
    ProcessMessageCount: 0
hekabench_counter:
    InChanCapacity: 50
    InChanLength: 0
    MatchChanCapacity: 50
    MatchChanLength: 0
    MatchAvgDuration: 445
    ProcessMessageCount: 0
    InjectMessageCount: 0
    Memory: 20644
    MaxMemory: 20644
    MaxInstructions: 18
    MaxOutput: 0
    ProcessMessageAvgDuration: 0
    TimerEventAvgDuration: 78532

====Outputs====
LogOutput:
    InChanCapacity: 50
    InChanLength: 0
    MatchChanCapacity: 50
    MatchChanLength: 0
    MatchAvgDuration: 406
DashboardOutput:
    InChanCapacity: 50
    InChanLength: 0
    MatchChanCapacity: 50
    MatchChanLength: 0
    MatchAvgDuration: 336
========
`

			c.Expect(report, gs.Equals, expected)
		})

		c.Specify("works w/ bad param config file", func() {
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_bad_params.toml")
			c.Assume(err, gs.Not(gs.IsNil))
		})

		c.Specify("works w/ common parameters that are not part of the struct", func() {
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_test_common.toml")
			c.Assume(err, gs.IsNil)

		})

	})

	c.Specify("Config directory helpers", func() {
		Globals().BaseDir = "/base/dir"
		Globals().ShareDir = "/share/dir"

		c.Specify("PrependBaseDir", func() {
			c.Specify("prepends for relative paths", func() {
				dir := filepath.FromSlash("relative/path")
				result := PrependBaseDir(dir)
				c.Expect(result, gs.Equals, filepath.FromSlash("/base/dir/relative/path"))
			})

			c.Specify("doesn't prepend for absolute paths", func() {
				dir := filepath.FromSlash("/absolute/path")
				result := PrependBaseDir(dir)
				c.Expect(result, gs.Equals, dir)
			})
		})

		c.Specify("PrependShareDir", func() {
			c.Specify("prepends for relative paths", func() {
				dir := filepath.FromSlash("relative/path")
				result := PrependShareDir(dir)
				c.Expect(result, gs.Equals, filepath.FromSlash("/share/dir/relative/path"))
			})

			c.Specify("doesn't prepend for absolute paths", func() {
				dir := filepath.FromSlash("/absolute/path")
				result := PrependShareDir(dir)
				c.Expect(result, gs.Equals, dir)
			})
		})
	})

	c.Specify("PluginHelper", func() {
		c.Specify("starts and stops DecoderRunners appropriately", func() {
			err := pipeConfig.LoadFromConfigFile("./testsupport/config_test.toml")
			c.Assume(err, gs.IsNil)
			// Start two DecoderRunners.
			dr1, ok := pipeConfig.DecoderRunner("ProtobufDecoder", "ProtobufDecoder_1")
			c.Expect(ok, gs.IsTrue)
			dr2, ok := pipeConfig.DecoderRunner("ProtobufDecoder", "ProtobufDecoder_2")
			c.Expect(ok, gs.IsTrue)
			// Stop the second one.
			ok = pipeConfig.StopDecoderRunner(dr2)
			c.Expect(ok, gs.IsTrue)
			// Verify that it's stopped, i.e. InChan is closed.
			_, ok = <-dr2.InChan()
			c.Expect(ok, gs.IsFalse)

			// Verify that dr1 is *not* stopped, i.e. InChan is still open.
			rChan := make(chan *PipelinePack, 1)
			pack := NewPipelinePack(rChan)
			dr1.InChan() <- pack // <-- Failure case means this will panic.

			// Try to stop dr2 again. Shouldn't fail, but ok should be false.
			ok = pipeConfig.StopDecoderRunner(dr2)
			c.Expect(ok, gs.IsFalse)

			// Clean up our UdpInput.
			udp, _ := pipeConfig.InputRunners["UdpInput"]
			udp.Input().Stop()
		})
	})
}
Example #26
0
func LoadFromConfigSpec(c gs.Context) {
	c.Specify("Config file loading", func() {
		origGlobals := Globals
		origDecodersByEncoding := DecodersByEncoding
		pipeConfig := NewPipelineConfig(nil)
		defer func() {
			Globals = origGlobals
			DecodersByEncoding = origDecodersByEncoding
		}()

		c.Assume(pipeConfig, gs.Not(gs.IsNil))

		c.Specify("works w/ good config file", func() {
			err := pipeConfig.LoadFromConfigFile("../testsupport/config_test.toml")
			c.Assume(err, gs.IsNil)

			// We use a set of Expect's rather than c.Specify because the
			// pipeConfig can't be re-loaded per child as gospec will do
			// since each one needs to bind to the same address

			// and the decoders are loaded for the right encoding headers
			dSet := pipeConfig.DecoderSets[0]
			dRunner, ok := dSet.ByEncoding(message.Header_JSON)
			c.Expect(dRunner, gs.Not(gs.IsNil))
			c.Expect(ok, gs.IsTrue)
			c.Expect(dRunner.Name(), gs.Equals, "JsonDecoder")

			dRunner, ok = dSet.ByEncoding(message.Header_PROTOCOL_BUFFER)
			c.Expect(dRunner, gs.Not(gs.IsNil))
			c.Expect(ok, gs.IsTrue)
			c.Expect(dRunner.Name(), gs.Equals, "ProtobufDecoder")

			// decoders channel is full
			c.Expect(len(pipeConfig.decodersChan), gs.Equals, Globals().DecoderPoolSize)

			// and the inputs section loads properly with a custom name
			_, ok = pipeConfig.InputRunners["UdpInput"]
			c.Expect(ok, gs.Equals, true)

			// and the decoders sections load
			_, ok = pipeConfig.DecoderWrappers["JsonDecoder"]
			c.Expect(ok, gs.Equals, true)
			_, ok = pipeConfig.DecoderWrappers["ProtobufDecoder"]
			c.Expect(ok, gs.Equals, true)

			// and the outputs section loads
			_, ok = pipeConfig.OutputRunners["LogOutput"]
			c.Expect(ok, gs.Equals, true)

			// and the filters sections loads
			_, ok = pipeConfig.FilterRunners["sample"]
			c.Expect(ok, gs.Equals, true)
		})

		c.Specify("works w/ decoder defaults", func() {
			err := pipeConfig.LoadFromConfigFile("../testsupport/config_test_defaults.toml")
			c.Assume(err, gs.Not(gs.IsNil))

			// Decoders are loaded
			c.Expect(len(pipeConfig.DecoderWrappers), gs.Equals, 2)
			c.Expect(DecodersByEncoding[message.Header_JSON], gs.Equals, "JsonDecoder")
			c.Expect(DecodersByEncoding[message.Header_PROTOCOL_BUFFER], gs.Equals,
				"ProtobufDecoder")
		})
		c.Specify("explodes w/ bad config file", func() {
			err := pipeConfig.LoadFromConfigFile("../testsupport/config_bad_test.toml")
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), ts.StringContains, "2 errors loading plugins")
			c.Expect(pipeConfig.logMsgs, gs.ContainsAny, gs.Values("No such plugin: CounterOutput"))
		})

		c.Specify("handles missing config file correctly", func() {
			err := pipeConfig.LoadFromConfigFile("no_such_file.toml")
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), ts.StringContains, "open no_such_file.toml: no such file or directory")
		})

		c.Specify("errors correctly w/ bad outputs config", func() {
			err := pipeConfig.LoadFromConfigFile("../testsupport/config_bad_outputs.toml")
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), ts.StringContains, "1 errors loading plugins")
			msg := pipeConfig.logMsgs[0]
			c.Expect(msg, ts.StringContains, "No such plugin")
		})

		c.Specify("captures plugin Init() panics", func() {
			RegisterPlugin("PanicOutput", func() interface{} {
				return new(PanicOutput)
			})
			err := pipeConfig.LoadFromConfigFile("../testsupport/config_panic.toml")
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
Example #27
0
func DecodersSpec(c gospec.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	msg := getTestMessage()
	config := NewPipelineConfig(nil)

	c.Specify("A ProtobufDecoder", func() {
		encoded, err := proto.Marshal(msg)
		c.Assume(err, gs.IsNil)
		pack := NewPipelinePack(config.inputRecycleChan)
		decoder := new(ProtobufDecoder)

		c.Specify("decodes a protobuf message", func() {
			pack.MsgBytes = encoded
			err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(pack.Message, gs.Equals, msg)
			v, ok := pack.Message.GetFieldValue("foo")
			c.Expect(ok, gs.IsTrue)
			c.Expect(v, gs.Equals, "bar")
		})

		c.Specify("returns an error for bunk encoding", func() {
			bunk := append([]byte{0, 0, 0}, encoded...)
			pack.MsgBytes = bunk
			err := decoder.Decode(pack)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})

	c.Specify("A MultiDecoder", func() {
		decoder := new(MultiDecoder)
		conf := decoder.ConfigStruct().(*MultiDecoderConfig)

		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		conf.Name = "MyMultiDecoder"
		conf.Subs = make(map[string]interface{}, 0)

		conf.Subs["StartsWithM"] = make(map[string]interface{}, 0)
		withM := conf.Subs["StartsWithM"].(map[string]interface{})
		withM["type"] = "PayloadRegexDecoder"
		withM["match_regex"] = "^(?P<TheData>m.*)"
		withMFields := make(map[string]interface{}, 0)
		withMFields["StartsWithM"] = "%TheData%"
		withM["message_fields"] = withMFields

		conf.Order = []string{"StartsWithM"}

		errMsg := "Unable to decode message with any contained decoder."

		dRunner := NewMockDecoderRunner(ctrl)
		// An error will be spit out b/c there's no real *dRunner in there;
		// doesn't impact the tests.
		dRunner.EXPECT().LogError(gomock.Any())

		c.Specify("decodes simple messages", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "matching text"
			pack.Message.SetPayload(regex_data)
			err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetType(), gs.Equals, "heka.MyMultiDecoder")
			value, ok := pack.Message.GetFieldValue("StartsWithM")
			c.Assume(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, regex_data)
		})

		c.Specify("returns an error if all decoders fail", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "non-matching text"
			pack.Message.SetPayload(regex_data)
			err = decoder.Decode(pack)
			c.Expect(err.Error(), gs.Equals, errMsg)
		})

		c.Specify("logs subdecoder failures when configured to do so", func() {
			conf.LogSubErrors = true
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "non-matching text"
			pack.Message.SetPayload(regex_data)

			// Expect that we log an error for undecoded message.
			dRunner.EXPECT().LogError(fmt.Errorf("Subdecoder 'StartsWithM' decode error: No match"))

			err = decoder.Decode(pack)
			c.Expect(err.Error(), gs.Equals, errMsg)
		})

		c.Specify("sets subdecoder runner correctly", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			// Call LogError to appease the angry gomock gods.
			dRunner.LogError(errors.New("foo"))

			// Now create a real *dRunner, pass it in, make sure a wrapper
			// gets handed to the subdecoder.
			dr := NewDecoderRunner(decoder.Name, decoder, new(PluginGlobals))
			decoder.SetDecoderRunner(dr)
			sub := decoder.Decoders["StartsWithM"]
			subRunner := sub.(*PayloadRegexDecoder).dRunner
			subRunner, ok := subRunner.(*mDRunner)
			c.Expect(ok, gs.IsTrue)
			c.Expect(subRunner.Name(), gs.Equals,
				fmt.Sprintf("%s-StartsWithM", decoder.Name))
			c.Expect(subRunner.Decoder(), gs.Equals, sub)
		})

		c.Specify("with multiple registered decoders", func() {
			conf.Subs["StartsWithS"] = make(map[string]interface{}, 0)
			withS := conf.Subs["StartsWithS"].(map[string]interface{})
			withS["type"] = "PayloadRegexDecoder"
			withS["match_regex"] = "^(?P<TheData>s.*)"
			withSFields := make(map[string]interface{}, 0)
			withSFields["StartsWithS"] = "%TheData%"
			withS["message_fields"] = withSFields

			conf.Subs["StartsWithM2"] = make(map[string]interface{}, 0)
			withM2 := conf.Subs["StartsWithM2"].(map[string]interface{})
			withM2["type"] = "PayloadRegexDecoder"
			withM2["match_regex"] = "^(?P<TheData>m.*)"
			withM2Fields := make(map[string]interface{}, 0)
			withM2Fields["StartsWithM2"] = "%TheData%"
			withM2["message_fields"] = withM2Fields

			conf.Order = append(conf.Order, "StartsWithS", "StartsWithM2")

			var ok bool

			// Two more subdecoders means two more LogError calls.
			dRunner.EXPECT().LogError(gomock.Any()).Times(2)

			c.Specify("defaults to `first-wins` cascading", func() {
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)

				c.Specify("on a first match condition", func() {
					regexData := "match first"
					pack.Message.SetPayload(regexData)
					err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("and a second match condition", func() {
					regexData := "second match"
					pack.Message.SetPayload(regexData)
					err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("returning an error if they all fail", func() {
					regexData := "won't match"
					pack.Message.SetPayload(regexData)
					err = decoder.Decode(pack)
					c.Expect(err.Error(), gs.Equals, errMsg)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})
			})

			c.Specify("and using `all` cascading", func() {
				conf.CascadeStrategy = "all"
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)

				c.Specify("matches multiples when appropriate", func() {
					regexData := "matches twice"
					pack.Message.SetPayload(regexData)
					err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsTrue)
				})

				c.Specify("matches singles when appropriate", func() {
					regexData := "second match"
					pack.Message.SetPayload(regexData)
					err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("returns an error if they all fail", func() {
					regexData := "won't match"
					pack.Message.SetPayload(regexData)
					err = decoder.Decode(pack)
					c.Expect(err.Error(), gs.Equals, errMsg)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})
			})
		})
	})

	c.Specify("A PayloadJsonDecoder", func() {
		decoder := new(PayloadJsonDecoder)
		conf := decoder.ConfigStruct().(*PayloadJsonDecoderConfig)
		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		c.Specify("decodes simple messages", func() {
			json_data := `{"statsd": {"count": 1, "name": "some.counter"}, "pid": 532, "timestamp": "2013-08-13T10:32:00.000Z"}`
			conf.JsonMap = map[string]string{"Count": "$.statsd.count",
				"Name":      "$.statsd.name",
				"Pid":       "$.pid",
				"Timestamp": "$.timestamp",
			}

			conf.MessageFields = MessageTemplate{
				"Pid":       "%Pid%",
				"StatCount": "%Count%",
				"StatName":  "%Name%",
				"Timestamp": "%Timestamp%",
			}
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)
			pack.Message.SetPayload(json_data)
			err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)
			c.Expect(pack.Message.GetPid(), gs.Equals, int32(532))

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1376389920000000000))

			var ok bool
			var name, count interface{}
			count, ok = pack.Message.GetFieldValue("StatCount")
			c.Expect(ok, gs.Equals, true)
			c.Expect(count, gs.Equals, "1")

			name, ok = pack.Message.GetFieldValue("StatName")
			c.Expect(ok, gs.Equals, true)
			c.Expect(name, gs.Equals, "some.counter")
		})
	})

	c.Specify("A PayloadXmlDecoder", func() {
		decoder := new(PayloadXmlDecoder)
		conf := decoder.ConfigStruct().(*PayloadXmlDecoderConfig)
		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		c.Specify("decodes simple messages", func() {
			xml_data := `<library>
    <!-- Great book. -->
    <book id="b0836217462" available="true">
        <isbn>0836217462</isbn>
        <title lang="en">Being a Dog Is a Full-Time Job</title>
        <quote>I'd dog paddle the deepest ocean.</quote>
        <author id="CMS">
            <?echo "go rocks"?>
            <name>Charles M Schulz</name>
            <born>1922-11-26</born>
            <dead>2000-02-12</dead>
        </author>
        <character id="PP">
            <name>Peppermint Patty</name>
            <born>1966-08-22</born>
            <qualificati>bold, brash and tomboyish</qualificati>
        </character>
        <character id="Snoopy">
            <name>Snoopy</name>
            <born>1950-10-04</born>
            <qualificati>extroverted beagle</qualificati>
        </character>
    </book>
</library>`

			conf.XPathMapConfig = map[string]string{"Isbn": "library/*/isbn",
				"Name":    "/library/book/character[born='1950-10-04']/name",
				"Patty":   "/library/book//node()[@id='PP']/name",
				"Title":   "//book[author/@id='CMS']/title",
				"Comment": "/library/book/preceding::comment()",
			}

			conf.MessageFields = MessageTemplate{
				"Isbn":    "%Isbn%",
				"Name":    "%Name%",
				"Patty":   "%Patty%",
				"Title":   "%Title%",
				"Comment": "%Comment%",
			}
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)
			pack.Message.SetPayload(xml_data)
			err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			var isbn, name, patty, title, comment interface{}
			var ok bool

			isbn, ok = pack.Message.GetFieldValue("Isbn")
			c.Expect(ok, gs.Equals, true)

			name, ok = pack.Message.GetFieldValue("Name")
			c.Expect(ok, gs.Equals, true)

			patty, ok = pack.Message.GetFieldValue("Patty")
			c.Expect(ok, gs.Equals, true)

			title, ok = pack.Message.GetFieldValue("Title")
			c.Expect(ok, gs.Equals, true)

			comment, ok = pack.Message.GetFieldValue("Comment")
			c.Expect(ok, gs.Equals, true)

			c.Expect(isbn, gs.Equals, "0836217462")
			c.Expect(name, gs.Equals, "Snoopy")
			c.Expect(patty, gs.Equals, "Peppermint Patty")
			c.Expect(title, gs.Equals, "Being a Dog Is a Full-Time Job")
			c.Expect(comment, gs.Equals, " Great book. ")
		})
	})

	c.Specify("A PayloadRegexDecoder", func() {
		decoder := new(PayloadRegexDecoder)
		conf := decoder.ConfigStruct().(*PayloadRegexDecoderConfig)
		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)
		conf.TimestampLayout = "02/Jan/2006:15:04:05 -0700"

		c.Specify("non capture regex", func() {
			conf.MatchRegex = `\d+`
			err := decoder.Init(conf)
			c.Expect(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), gs.Equals, "PayloadRegexDecoder regex must contain capture groups")
		})

		c.Specify("invalid regex", func() {
			conf.MatchRegex = `\mtest`
			err := decoder.Init(conf)
			c.Expect(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), gs.Equals, "PayloadRegexDecoder: error parsing regexp: invalid escape sequence: `\\m`")
		})

		c.Specify("reading an apache timestamp", func() {
			conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]`
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)
			pack.Message.SetPayload("[18/Apr/2013:14:00:28 -0700]")
			err = decoder.Decode(pack)
			c.Expect(pack.Message.GetTimestamp(), gs.Equals, int64(1366318828000000000))
			pack.Zero()
		})

		c.Specify("uses kitchen timestamp", func() {
			conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]`
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)
			pack.Message.SetPayload("[5:16PM]")
			now := time.Now()
			cur_date := time.Date(now.Year(), now.Month(), now.Day(), 17, 16, 0, 0,
				time.UTC)
			err = decoder.Decode(pack)
			c.Expect(pack.Message.GetTimestamp(), gs.Equals, cur_date.UnixNano())
			pack.Zero()
		})

		c.Specify("adjusts timestamps as specified", func() {
			conf.MatchRegex = `\[(?P<Timestamp>[^\]]+)\]`
			conf.TimestampLayout = "02/Jan/2006:15:04:05"
			conf.TimestampLocation = "America/Los_Angeles"
			timeStr := "18/Apr/2013:14:00:28"
			loc, err := time.LoadLocation(conf.TimestampLocation)
			c.Assume(err, gs.IsNil)
			expectedLocal, err := time.ParseInLocation(conf.TimestampLayout, timeStr, loc)
			c.Assume(err, gs.IsNil)
			err = decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)
			pack.Message.SetPayload("[" + timeStr + "]")
			err = decoder.Decode(pack)
			c.Expect(pack.Message.GetTimestamp(), gs.Equals, expectedLocal.UnixNano())
			pack.Zero()
		})

		c.Specify("apply representation metadata to a captured field", func() {
			value := "0.23"
			payload := "header"
			conf.MatchRegex = `(?P<ResponseTime>\d+\.\d+)`
			conf.MessageFields = MessageTemplate{
				"ResponseTime|s": "%ResponseTime%",
				"Payload|s":      "%ResponseTime%",
				"Payload":        payload,
			}
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)
			pack.Message.SetPayload(value)
			err = decoder.Decode(pack)

			f := pack.Message.FindFirstField("ResponseTime")
			c.Expect(f, gs.Not(gs.IsNil))
			c.Expect(f.GetValue(), gs.Equals, value)
			c.Expect(f.GetRepresentation(), gs.Equals, "s")

			f = pack.Message.FindFirstField("Payload")
			c.Expect(f, gs.Not(gs.IsNil))
			c.Expect(f.GetValue(), gs.Equals, value)
			c.Expect(f.GetRepresentation(), gs.Equals, "s")

			c.Expect(pack.Message.GetPayload(), gs.Equals, payload)

			pack.Zero()
		})

		c.Specify("reading test-zeus.log", func() {
			conf.MatchRegex = `(?P<Ip>([0-9]{1,3}\.){3}[0-9]{1,3}) (?P<Hostname>(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])) (?P<User>\w+) \[(?P<Timestamp>[^\]]+)\] \"(?P<Verb>[A-X]+) (?P<Request>\/\S*) HTTP\/(?P<Httpversion>\d\.\d)\" (?P<Response>\d{3}) (?P<Bytes>\d+)`
			conf.MessageFields = MessageTemplate{
				"hostname": "%Hostname%",
				"ip":       "%Ip%",
				"response": "%Response%",
			}
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			filePath := "../testsupport/test-zeus.log"
			fileBytes, err := ioutil.ReadFile(filePath)
			c.Assume(err, gs.IsNil)
			fileStr := string(fileBytes)
			lines := strings.Split(fileStr, "\n")

			containsFieldValue := func(str, fieldName string, msg *message.Message) bool {
				raw, ok := msg.GetFieldValue(fieldName)
				if !ok {
					return false
				}
				value := raw.(string)
				return strings.Contains(str, value)
			}

			c.Specify("extracts capture data and puts it in the message fields", func() {
				var misses int
				for _, line := range lines {
					if strings.TrimSpace(line) == "" {
						continue
					}
					pack.Message.SetPayload(line)
					err = decoder.Decode(pack)
					if err != nil {
						misses++
						continue
					}
					c.Expect(containsFieldValue(line, "hostname", pack.Message), gs.IsTrue)
					c.Expect(containsFieldValue(line, "ip", pack.Message), gs.IsTrue)
					c.Expect(containsFieldValue(line, "response", pack.Message), gs.IsTrue)
					pack.Zero()
				}
				c.Expect(misses, gs.Equals, 3)
			})
		})

		c.Specify("reading test-severity.log", func() {
			conf.MatchRegex = `severity: (?P<Severity>[a-zA-Z]+)`
			conf.SeverityMap = map[string]int32{
				"emergency": 0,
				"alert":     1,
				"critical":  2,
				"error":     3,
				"warning":   4,
				"notice":    5,
				"info":      6,
				"debug":     7,
			}
			reverseMap := make(map[int32]string)
			for str, i := range conf.SeverityMap {
				reverseMap[i] = str
			}
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)

			filePath := "../testsupport/test-severity.log"
			fileBytes, err := ioutil.ReadFile(filePath)
			c.Assume(err, gs.IsNil)
			fileStr := string(fileBytes)
			lines := strings.Split(fileStr, "\n")

			c.Specify("sets message severity based on SeverityMap", func() {
				err := errors.New("Don't recognize severity: 'BOGUS'")
				dRunner.EXPECT().LogError(err)
				for _, line := range lines {
					if strings.TrimSpace(line) == "" {
						continue
					}
					pack.Message.SetPayload(line)
					err = decoder.Decode(pack)
					if err != nil {
						fmt.Println(line)
					}
					c.Expect(err, gs.IsNil)
					if strings.Contains(line, "BOGUS") {
						continue
					}
					strVal := reverseMap[pack.Message.GetSeverity()]
					c.Expect(strings.Contains(line, strVal), gs.IsTrue)
				}
			})
		})
	})

	c.Specify("A SandboxDecoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../sandbox/lua/testsupport/decoder.lua"
		conf.ScriptType = "lua"
		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		c.Specify("decodes simple messages", func() {
			data := "1376389920 debug id=2321 url=example.com item=1"
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)
			pack.Message.SetPayload(data)
			err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1376389920000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("id")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "2321")

			value, ok = pack.Message.GetFieldValue("url")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "example.com")

			value, ok = pack.Message.GetFieldValue("item")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "1")
		})

		c.Specify("decodes an invalid messages", func() {
			data := "1376389920 bogus id=2321 url=example.com item=1"
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			dRunner := NewMockDecoderRunner(ctrl)
			decoder.SetDecoderRunner(dRunner)
			pack.Message.SetPayload(data)
			err = decoder.Decode(pack)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
		})
	})

	c.Specify("A StatsToFieldsDecoder", func() {
		decoder := new(StatsToFieldsDecoder)
		router := NewMessageRouter()
		router.inChan = make(chan *PipelinePack, 5)
		dRunner := NewMockDecoderRunner(ctrl)
		decoder.runner = dRunner
		dRunner.EXPECT().Router().Return(router)

		pack := NewPipelinePack(config.inputRecycleChan)

		mergeStats := func(stats [][]string) string {
			lines := make([]string, len(stats))
			for i, line := range stats {
				lines[i] = strings.Join(line, " ")
			}
			return strings.Join(lines, "\n")
		}

		c.Specify("correctly converts stats to fields", func() {
			stats := [][]string{
				{"stat.one", "1", "1380047333"},
				{"stat.two", "2", "1380047333"},
				{"stat.three", "3", "1380047333"},
				{"stat.four", "4", "1380047333"},
				{"stat.five", "5", "1380047333"},
			}
			pack.Message.SetPayload(mergeStats(stats))
			err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)

			for i, stats := range stats {
				value, ok := pack.Message.GetFieldValue(stats[0])
				c.Expect(ok, gs.IsTrue)
				expected := float64(i + 1)
				c.Expect(value.(float64), gs.Equals, expected)
			}

			value, ok := pack.Message.GetFieldValue("timestamp")
			c.Expect(ok, gs.IsTrue)
			expected, err := strconv.ParseInt(stats[0][2], 0, 32)
			c.Assume(err, gs.IsNil)
			c.Expect(value.(int64), gs.Equals, expected)
		})

		c.Specify("generates multiple messages for multiple timestamps", func() {
			stats := [][]string{
				{"stat.one", "1", "1380047333"},
				{"stat.two", "2", "1380047333"},
				{"stat.three", "3", "1380047331"},
				{"stat.four", "4", "1380047333"},
				{"stat.five", "5", "1380047332"},
			}
			// Prime the pack supply w/ two new packs.
			dRunner.EXPECT().NewPack().Return(NewPipelinePack(nil))
			dRunner.EXPECT().NewPack().Return(NewPipelinePack(nil))

			// Decode and check the main pack.
			pack.Message.SetPayload(mergeStats(stats))
			err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			value, ok := pack.Message.GetFieldValue("timestamp")
			c.Expect(ok, gs.IsTrue)
			expected, err := strconv.ParseInt(stats[0][2], 0, 32)
			c.Assume(err, gs.IsNil)
			c.Expect(value.(int64), gs.Equals, expected)

			// Check the first extra.
			pack = <-router.inChan
			value, ok = pack.Message.GetFieldValue("timestamp")
			c.Expect(ok, gs.IsTrue)
			expected, err = strconv.ParseInt(stats[2][2], 0, 32)
			c.Assume(err, gs.IsNil)
			c.Expect(value.(int64), gs.Equals, expected)

			// Check the second extra.
			pack = <-router.inChan
			value, ok = pack.Message.GetFieldValue("timestamp")
			c.Expect(ok, gs.IsTrue)
			expected, err = strconv.ParseInt(stats[4][2], 0, 32)
			c.Assume(err, gs.IsNil)
			c.Expect(value.(int64), gs.Equals, expected)
		})

		c.Specify("fails w/ invalid timestamp", func() {
			stats := [][]string{
				{"stat.one", "1", "1380047333"},
				{"stat.two", "2", "1380047333"},
				{"stat.three", "3", "1380047333c"},
				{"stat.four", "4", "1380047333"},
				{"stat.five", "5", "1380047332"},
			}
			pack.Message.SetPayload(mergeStats(stats))
			err := decoder.Decode(pack)
			c.Expect(err, gs.Not(gs.IsNil))
			expected := fmt.Sprintf("invalid timestamp: '%s'",
				strings.Join(stats[2], " "))
			c.Expect(err.Error(), gs.Equals, expected)
		})

		c.Specify("fails w/ invalid value", func() {
			stats := [][]string{
				{"stat.one", "1", "1380047333"},
				{"stat.two", "2", "1380047333"},
				{"stat.three", "3", "1380047333"},
				{"stat.four", "4d", "1380047333"},
				{"stat.five", "5", "1380047332"},
			}
			pack.Message.SetPayload(mergeStats(stats))
			err := decoder.Decode(pack)
			c.Expect(err, gs.Not(gs.IsNil))
			expected := fmt.Sprintf("invalid value: '%s'",
				strings.Join(stats[3], " "))
			c.Expect(err.Error(), gs.Equals, expected)
		})
	})
}
Example #28
0
func MessageEqualsSpec(c gospec.Context) {
	msg0 := getTestMessage()

	c.Specify("Messages are equal", func() {
		msg1 := CopyMessage(msg0)
		c.Expect(msg0, gs.Equals, msg1)
	})

	c.Specify("Messages w/ diff severity", func() {
		msg1 := CopyMessage(msg0)
		*msg1.Severity--
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ diff uuid", func() {
		msg1 := CopyMessage(msg0)
		u := uuid.NewRandom()
		copy(msg1.Uuid, u)
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ diff payload", func() {
		msg1 := CopyMessage(msg0)
		*msg1.Payload = "Something completely different"
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ diff number of fields", func() {
		msg1 := CopyMessage(msg0)
		f, _ := NewField("sna", "foo", "")
		msg1.AddField(f)
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ diff number of field values in a key", func() {
		msg1 := CopyMessage(msg0)
		f := msg1.FindFirstField("foo")
		f.AddValue("foo1")
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ diff value in a field", func() {
		msg1 := CopyMessage(msg0)
		f := msg1.FindFirstField("foo")
		f.ValueString[0] = "bah"
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ diff field key", func() {
		msg1 := CopyMessage(msg0)
		f := msg1.FindFirstField("foo")
		*f.Name = "widget"
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})

	c.Specify("Messages w/ recurring keys", func() {
		msg0 = &Message{}
		f, _ := NewField("foo", "bar", "")
		f1, _ := NewField("foo", "bar1", "")
		msg0.AddField(f)
		msg0.AddField(f1)
		msg1 := CopyMessage(msg0)
		c.Expect(msg0, gs.Equals, msg1)
		foos := msg0.FindAllFields("foo")
		foos[1].ValueString[0] = "bar2"
		c.Expect(msg0, gs.Not(gs.Equals), msg1)
	})
}
Example #29
0
func FilehandlingSpec(c gs.Context) {
	here, err := os.Getwd()
	c.Assume(err, gs.IsNil)
	dirPath := filepath.Join(here, "testdir", "filehandling")

	c.Specify("The directory scanner", func() {

		c.Specify("scans a directory properly", func() {
			matchRegex := regexp.MustCompile(dirPath + `/subdir/.*\.log(\..*)?`)
			results := ScanDirectoryForLogfiles(dirPath, matchRegex)
			c.Expect(len(results), gs.Equals, 3)
		})

		c.Specify("scans a directory with a bad regexp", func() {
			matchRegex := regexp.MustCompile(dirPath + "/subdir/.*.logg(.*)?")
			results := ScanDirectoryForLogfiles(dirPath, matchRegex)
			c.Expect(len(results), gs.Equals, 0)
		})
	})

	c.Specify("Populating logfile with match parts", func() {
		logfile := Logfile{}

		c.Specify("works without errors", func() {
			subexpNames := []string{"MonthName", "LogNumber"}
			matches := []string{"October", "24"}
			translation := make(SubmatchTranslationMap)
			logfile.PopulateMatchParts(subexpNames, matches, translation)
			c.Expect(logfile.MatchParts["MonthName"], gs.Equals, 10)
			c.Expect(logfile.MatchParts["LogNumber"], gs.Equals, 24)
		})

		c.Specify("works with bad month name", func() {
			subexpNames := []string{"MonthName", "LogNumber"}
			matches := []string{"Octoberrr", "24"}
			translation := make(SubmatchTranslationMap)
			err := logfile.PopulateMatchParts(subexpNames, matches, translation)
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), gs.Equals, "Unable to locate month name: Octoberrr")
		})

		c.Specify("works with missing value in submatch translation map", func() {
			subexpNames := []string{"MonthName", "LogNumber"}
			matches := []string{"October", "24"}
			translation := make(SubmatchTranslationMap)
			translation["LogNumber"] = make(MatchTranslationMap)
			translation["LogNumber"]["23"] = 22
			translation["LogNumber"]["999"] = 999 // Non-"missing" submatches must be len > 1.
			err := logfile.PopulateMatchParts(subexpNames, matches, translation)
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), gs.Equals, "Value '24' not found in translation map 'LogNumber'.")
		})

		c.Specify("works with custom value in submatch translation map", func() {
			subexpNames := []string{"MonthName", "LogNumber"}
			matches := []string{"October", "24"}
			translation := make(SubmatchTranslationMap)
			translation["LogNumber"] = make(MatchTranslationMap)
			translation["LogNumber"]["24"] = 2
			translation["LogNumber"]["999"] = 999 // Non-"missing" submatches must be len > 1.
			logfile.PopulateMatchParts(subexpNames, matches, translation)
			c.Expect(logfile.MatchParts["MonthName"], gs.Equals, 10)
			c.Expect(logfile.MatchParts["LogNumber"], gs.Equals, 2)
		})
	})

	c.Specify("Populating logfiles with match parts", func() {
		translation := make(SubmatchTranslationMap)
		matchRegex := regexp.MustCompile(dirPath + `/subdir/.*\.log\.?(?P<FileNumber>.*)?`)
		logfiles := ScanDirectoryForLogfiles(dirPath, matchRegex)

		c.Specify("is populated", func() {
			logfiles.PopulateMatchParts(matchRegex, translation)
			c.Expect(len(logfiles), gs.Equals, 3)
			c.Expect(logfiles[0].MatchParts["FileNumber"], gs.Equals, -1)
			c.Expect(logfiles[1].MatchParts["FileNumber"], gs.Equals, 1)
			c.Expect(logfiles[1].StringMatchParts["FileNumber"], gs.Equals, "1")
		})

		c.Specify("returns errors", func() {
			translation["FileNumber"] = make(MatchTranslationMap)
			translation["FileNumber"]["23"] = 22
			translation["FileNumber"]["999"] = 999 // Non-"missing" submatches must be len > 1.
			err := logfiles.PopulateMatchParts(matchRegex, translation)
			c.Assume(err, gs.Not(gs.IsNil))
			c.Expect(len(logfiles), gs.Equals, 3)
		})
	})

	c.Specify("Sorting logfiles", func() {
		translation := make(SubmatchTranslationMap)
		matchRegex := regexp.MustCompile(dirPath + `/subdir/.*\.log\.?(?P<FileNumber>.*)?`)
		logfiles := ScanDirectoryForLogfiles(dirPath, matchRegex)

		c.Specify("with no 'missing' translation value", func() {
			err := logfiles.PopulateMatchParts(matchRegex, translation)
			c.Assume(err, gs.IsNil)
			c.Expect(len(logfiles), gs.Equals, 3)

			c.Specify("can be sorted newest to oldest", func() {
				byp := ByPriority{Logfiles: logfiles, Priority: []string{"FileNumber"}}
				sort.Sort(byp)
				c.Expect(logfiles[0].MatchParts["FileNumber"], gs.Equals, -1)
				c.Expect(logfiles[1].MatchParts["FileNumber"], gs.Equals, 1)
			})

			c.Specify("can be sorted oldest to newest", func() {
				byp := ByPriority{Logfiles: logfiles, Priority: []string{"^FileNumber"}}
				sort.Sort(byp)
				c.Expect(logfiles[0].MatchParts["FileNumber"], gs.Equals, 2)
				c.Expect(logfiles[1].MatchParts["FileNumber"], gs.Equals, 1)
			})
		})

		c.Specify("with 'missing' translation value", func() {
			translation["FileNumber"] = make(MatchTranslationMap)
			translation["FileNumber"]["missing"] = 5
			err := logfiles.PopulateMatchParts(matchRegex, translation)
			c.Assume(err, gs.IsNil)
			c.Expect(len(logfiles), gs.Equals, 3)

			c.Specify("honors 'missing' translation value", func() {
				byp := ByPriority{Logfiles: logfiles, Priority: []string{"FileNumber"}}
				sort.Sort(byp)
				c.Expect(logfiles[0].MatchParts["FileNumber"], gs.Equals, 1)
				c.Expect(logfiles[1].MatchParts["FileNumber"], gs.Equals, 2)
				c.Expect(logfiles[2].MatchParts["FileNumber"], gs.Equals, 5)
			})
		})
	})

	c.Specify("Sorting out a directory of access/error logs", func() {
		translation := make(SubmatchTranslationMap)
		matchRegex := regexp.MustCompile(dirPath +
			`/(?P<Year>\d+)/(?P<Month>\d+)/(?P<Type>\w+)\.log(\.(?P<Seq>\d+))?`)
		logfiles := ScanDirectoryForLogfiles(dirPath, matchRegex)
		err := logfiles.PopulateMatchParts(matchRegex, translation)
		c.Assume(err, gs.IsNil)
		c.Expect(len(logfiles), gs.Equals, 52)

		c.Specify("can result in multiple logfile streams", func() {
			mfs := FilterMultipleStreamFiles(logfiles, []string{"Type"})
			c.Expect(len(mfs), gs.Equals, 2)
			access, ok := mfs["access"]
			c.Assume(ok, gs.IsTrue)
			c.Expect(len(access), gs.Equals, 26)
			error, ok := mfs["error"]
			c.Assume(ok, gs.IsTrue)
			c.Expect(len(error), gs.Equals, 26)

			c.Specify("can be individually sorted properly by access", func() {
				byp := ByPriority{Logfiles: mfs["access"], Priority: []string{"Year", "Month", "^Seq"}}
				sort.Sort(byp)
				lf := mfs["access"]
				c.Expect(lf[0].FileName, gs.Equals, dirPath+"/2010/05/access.log.3")
				c.Expect(lf[len(lf)-1].FileName, gs.Equals, dirPath+"/2013/08/access.log")
			})

			c.Specify("can be individually sorted properly by error", func() {
				byp := ByPriority{Logfiles: mfs["error"], Priority: []string{"Year", "Month", "^Seq"}}
				sort.Sort(byp)
				lf := mfs["error"]
				c.Expect(lf[0].FileName, gs.Equals, dirPath+"/2010/07/error.log.2")
				c.Expect(lf[len(lf)-1].FileName, gs.Equals, dirPath+"/2013/08/error.log")
			})
		})

		c.Specify("Can result in multiple logfile streams with a prefix", func() {
			mfs := FilterMultipleStreamFiles(logfiles, []string{"website-", "Type"})
			c.Expect(len(mfs), gs.Equals, 2)
			access, ok := mfs["website-access"]
			c.Assume(ok, gs.IsTrue)
			c.Expect(len(access), gs.Equals, 26)
			error, ok := mfs["website-error"]
			c.Assume(ok, gs.IsTrue)
			c.Expect(len(error), gs.Equals, 26)
		})
	})
}
func StatAccumInputSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	c.Specify("A StatAccumInput", func() {
		statAccumInput := StatAccumInput{}
		config := statAccumInput.ConfigStruct().(*StatAccumInputConfig)
		pConfig := NewPipelineConfig(nil)
		statAccumInput.pConfig = pConfig

		c.Specify("ticker interval is zero", func() {
			config.TickerInterval = 0
			err := statAccumInput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
			expected := "TickerInterval must be greater than 0."
			c.Expect(err.Error(), gs.Equals, expected)
		})

		c.Specify("validates that data is emitted", func() {
			config.EmitInPayload = false
			err := statAccumInput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
			expected := "One of either `EmitInPayload` or `EmitInFields` must be set to true."
			c.Expect(err.Error(), gs.Equals, expected)
		})

		c.Specify("that is started", func() {
			ith := new(InputTestHelper)
			ith.MockHelper = NewMockPluginHelper(ctrl)
			ith.MockInputRunner = NewMockInputRunner(ctrl)
			ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
			ith.PackSupply = make(chan *PipelinePack, 1)
			ith.PackSupply <- ith.Pack

			tickChan := make(chan time.Time)
			var inputStarted sync.WaitGroup

			runErrChan := make(chan error)
			startInput := func() {
				inputStarted.Add(1)

				// A call to Ticker() is the last step in the input's startup before
				// it's ready to process packs. Any tests that need to pause until
				// this happens can use `inputStarted.Wait()`.
				ith.MockInputRunner.EXPECT().Ticker().Do(func() {
					inputStarted.Done()
				}).Return(tickChan)

				go func() {
					err := statAccumInput.Run(ith.MockInputRunner, ith.MockHelper)
					runErrChan <- err
				}()
			}

			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
			// Need one of these for every message delivered.
			ith.MockInputRunner.EXPECT().Name().Return("StatAccumInput")

			deliverCall := ith.MockInputRunner.EXPECT().Deliver(ith.Pack)
			var deliverCalled sync.WaitGroup
			deliverCalled.Add(1)
			deliverCall.Do(func(pack *PipelinePack) {
				deliverCalled.Done()
			})

			c.Specify("using normal namespaces", func() {
				config.EmitInFields = true
				config.EmitInPayload = false
				err := statAccumInput.Init(config)
				c.Expect(err, gs.IsNil)

				finalizeSendingStats := func() (*message.Message, error) {
					close(statAccumInput.statChan)
					err := <-runErrChan
					return ith.Pack.Message, err
				}

				sendTimer := func(key string, vals ...int) {
					for _, v := range vals {
						statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "ms", float32(1)}
					}
				}
				sendCounter := func(key string, vals ...int) {
					for _, v := range vals {
						statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "c", float32(1)}
					}
				}
				sendGauge := func(key string, vals ...int) {
					for _, v := range vals {
						statAccumInput.statChan <- Stat{key, strconv.Itoa(v), "g", float32(1)}
					}
				}

				drainStats := func() {
					ok := true
					for ok {
						if len(statAccumInput.statChan) > 0 {
							time.Sleep(100 * time.Millisecond)
						} else {
							ok = false
						}
					}
				}

				validateValueAtKey := func(msg *message.Message, key string, value interface{}) {
					fieldValue, ok := msg.GetFieldValue(key)
					if !ok {
						log.Printf("%s field is missing from the message\n", key)
					}
					c.Expect(ok, gs.IsTrue)
					if fieldValue != value {
						log.Printf("%s should be %v is %v\n", key, value, fieldValue)
					}
					c.Expect(fieldValue, gs.Equals, value)
				}

				c.Specify("emits timer with correct prefixes", func() {
					startInput()
					sendTimer("sample.timer", 10, 10, 20, 20)
					sendTimer("sample2.timer", 10)
					msg, err := finalizeSendingStats()
					c.Assume(err, gs.Equals, nil)

					validateValueAtKey(msg, "stats.timers.sample.timer.count", int64(4))
					validateValueAtKey(msg, "stats.timers.sample.timer.count_ps", 0.4)
					validateValueAtKey(msg, "stats.timers.sample.timer.mean", 15.0)
					validateValueAtKey(msg, "stats.timers.sample.timer.lower", 10.0)
					validateValueAtKey(msg, "stats.timers.sample.timer.upper", 20.0)
					validateValueAtKey(msg, "stats.timers.sample.timer.sum", 60.0)
					validateValueAtKey(msg, "stats.timers.sample.timer.mean_90", 15.0)
					validateValueAtKey(msg, "stats.timers.sample.timer.upper_90", 20.0)
					validateValueAtKey(msg, "stats.timers.sample2.timer.count", int64(1))
					validateValueAtKey(msg, "stats.timers.sample2.timer.count_ps", 0.1)
					validateValueAtKey(msg, "stats.timers.sample2.timer.mean", 10.0)
					validateValueAtKey(msg, "stats.timers.sample2.timer.lower", 10.0)
					validateValueAtKey(msg, "stats.timers.sample2.timer.upper", 10.0)
					validateValueAtKey(msg, "stats.timers.sample2.timer.sum", 10.0)
					validateValueAtKey(msg, "stats.timers.sample2.timer.mean_90", 10.0)
					validateValueAtKey(msg, "stats.timers.sample2.timer.upper_90", 10.0)

					validateValueAtKey(msg, "stats.statsd.numStats", int64(2))
				})

				c.Specify("emits counters with correct prefixes", func() {
					startInput()
					sendCounter("sample.cnt", 1, 2, 3, 4, 5)
					sendCounter("sample2.cnt", 159, 951)
					msg, err := finalizeSendingStats()
					c.Assume(err, gs.IsNil)

					validateValueAtKey(msg, "stats.counters.sample.cnt.count", int64(15))
					validateValueAtKey(msg, "stats.counters.sample.cnt.rate", 1.5)

					validateValueAtKey(msg, "stats.counters.sample2.cnt.count", int64(1110))
					validateValueAtKey(msg, "stats.counters.sample2.cnt.rate", 1110.0/float64(config.TickerInterval))
				})

				c.Specify("emits gauge with correct prefixes", func() {
					startInput()
					sendGauge("sample.gauge", 1, 2)
					sendGauge("sample2.gauge", 1, 2, 3, 4, 5)
					msg, err := finalizeSendingStats()
					c.Assume(err, gs.IsNil)
					validateValueAtKey(msg, "stats.gauges.sample.gauge", float64(2))
					validateValueAtKey(msg, "stats.gauges.sample2.gauge", float64(5))
				})

				c.Specify("emits correct statsd.numStats count", func() {
					startInput()
					sendGauge("sample.gauge", 1, 2)
					sendGauge("sample2.gauge", 1, 2)
					sendCounter("sample.cnt", 1, 2, 3, 4, 5)
					sendCounter("sample2.cnt", 159, 951)
					sendTimer("sample.timer", 10, 10, 20, 20)
					sendTimer("sample2.timer", 10, 20)
					msg, err := finalizeSendingStats()
					c.Assume(err, gs.IsNil)
					validateValueAtKey(msg, "stats.statsd.numStats", int64(6))
				})

				c.Specify("emits proper idle stats", func() {
					startInput()
					inputStarted.Wait()
					sendGauge("sample.gauge", 1, 2)
					sendCounter("sample.cnt", 1, 2, 3, 4, 5)
					sendTimer("sample.timer", 10, 10, 20, 20)
					drainStats()
					tickChan <- time.Now()

					deliverCalled.Wait()
					ith.Pack.Recycle(nil)
					ith.PackSupply <- ith.Pack
					ith.MockInputRunner.EXPECT().Name().Return("StatAccumInput")
					ith.MockInputRunner.EXPECT().Deliver(ith.Pack)

					msg, err := finalizeSendingStats()
					c.Assume(err, gs.IsNil)
					validateValueAtKey(msg, "stats.gauges.sample.gauge", float64(2))
					validateValueAtKey(msg, "stats.counters.sample.cnt.count", int64(0))
					validateValueAtKey(msg, "stats.timers.sample.timer.count", int64(0))
					validateValueAtKey(msg, "stats.statsd.numStats", int64(3))
				})

				c.Specify("omits idle stats", func() {
					config.DeleteIdleStats = true
					err := statAccumInput.Init(config)
					c.Assume(err, gs.IsNil)

					startInput()
					inputStarted.Wait() // Can't flush until the input has started.
					sendGauge("sample.gauge", 1, 2)
					sendCounter("sample.cnt", 1, 2, 3, 4, 5)
					sendTimer("sample.timer", 10, 10, 20, 20)
					drainStats()
					tickChan <- time.Now()
					deliverCalled.Wait()

					sendTimer("sample2.timer", 10, 20)
					drainStats()
					ith.Pack.Recycle(nil)
					ith.PackSupply <- ith.Pack
					ith.MockInputRunner.EXPECT().Name().Return("StatAccumInput")
					ith.MockInputRunner.EXPECT().Deliver(ith.Pack)
					msg, err := finalizeSendingStats()
					c.Assume(err, gs.IsNil)
					validateValueAtKey(msg, "stats.statsd.numStats", int64(1))
				})

			})

			c.Specify("using legacy namespaces", func() {
				config.LegacyNamespaces = true

				statName := "sample.stat"
				statVal := int64(303)
				testStat := Stat{statName, strconv.Itoa(int(statVal)), "c", float32(1)}

				validateMsgFields := func(msg *message.Message) {
					c.Expect(len(msg.Fields), gs.Equals, 4)

					// timestamp
					_, ok := msg.GetFieldValue("timestamp")
					c.Expect(ok, gs.IsTrue)

					var tmp interface{}
					var intTmp int64

					// stats.sample.stat
					tmp, ok = msg.GetFieldValue("stats." + statName)
					c.Expect(ok, gs.IsTrue)
					intTmp, ok = tmp.(int64)
					c.Expect(ok, gs.IsTrue)
					c.Expect(intTmp, gs.Equals, int64(30))

					// stats_counts.sample.stat
					tmp, ok = msg.GetFieldValue("stats_counts." + statName)
					c.Expect(ok, gs.IsTrue)
					intTmp, ok = tmp.(int64)
					c.Expect(ok, gs.IsTrue)
					c.Expect(intTmp, gs.Equals, statVal)

					// statsd.numStats
					tmp, ok = msg.GetFieldValue("statsd.numStats")

					c.Expect(ok, gs.IsTrue)
					intTmp, ok = tmp.(int64)
					c.Expect(ok, gs.IsTrue)
					c.Expect(intTmp, gs.Equals, int64(1))
				}

				validateMsgPayload := func(msg *message.Message) {
					lines := strings.Split(msg.GetPayload(), "\n")
					c.Expect(len(lines), gs.Equals, 4)
					c.Expect(lines[3], gs.Equals, "")

					var timestamp string
					for i := 0; i < 3; i++ {
						line := strings.Split(lines[i], " ")
						c.Expect(len(line), gs.Equals, 3)
						switch i {
						case 0:
							c.Expect(line[0], gs.Equals, "stats."+statName)
							c.Expect(line[1], gs.Equals, "30.300000")
							timestamp = line[2]
						case 1:
							c.Expect(line[0], gs.Equals, "stats_counts."+statName)
							c.Expect(line[1], gs.Equals, strconv.Itoa(int(statVal)))
							c.Expect(line[2], gs.Equals, timestamp)
						case 2:
							c.Expect(line[0], gs.Equals, "statsd.numStats")
							c.Expect(line[1], gs.Equals, "1")
							c.Expect(line[2], gs.Equals, timestamp)
						}
					}
					expected := strings.Join(lines, "\n")
					c.Expect(msg.GetPayload(), gs.Equals, expected)

				}

				c.Specify("emits data in payload by default", func() {
					err := statAccumInput.Init(config)
					c.Assume(err, gs.IsNil)

					startInput()
					statAccumInput.statChan <- testStat
					close(statAccumInput.statChan)
					err = <-runErrChan
					c.Expect(err, gs.IsNil)

					validateMsgPayload(ith.Pack.Message)
				})

				c.Specify("emits data in fields when specified", func() {
					config.EmitInFields = true
					err := statAccumInput.Init(config)
					c.Assume(err, gs.IsNil)

					startInput()
					statAccumInput.statChan <- testStat
					close(statAccumInput.statChan)
					err = <-runErrChan
					c.Expect(err, gs.IsNil)
					validateMsgFields(ith.Pack.Message)
					validateMsgPayload(ith.Pack.Message)
				})

				c.Specify("omits data in payload when specified", func() {
					config.EmitInPayload = false
					config.EmitInFields = true
					err := statAccumInput.Init(config)
					c.Assume(err, gs.IsNil)

					startInput()
					statAccumInput.statChan <- testStat
					close(statAccumInput.statChan)
					err = <-runErrChan
					c.Expect(err, gs.IsNil)

					validateMsgFields(ith.Pack.Message)
					c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "")
				})

				c.Specify("honors time ticker to flush", func() {
					err := statAccumInput.Init(config)
					c.Assume(err, gs.IsNil)

					startInput()
					inputStarted.Wait()

					statAccumInput.statChan <- testStat
					// Sleep until the stat is processed.
					for len(statAccumInput.statChan) > 0 {
						time.Sleep(50)
					}
					tickChan <- time.Now()

					// Don't try to validate our message payload until Deliver has
					// been called.
					deliverCalled.Wait()
					validateMsgPayload(ith.Pack.Message)

					// Prep pack and EXPECTS for the close.
					ith.PackSupply <- ith.Pack
					ith.MockInputRunner.EXPECT().Name().Return("StatAccumInput")
					ith.MockInputRunner.EXPECT().Deliver(ith.Pack)

					close(statAccumInput.statChan)
					err = <-runErrChan
					c.Expect(err, gs.IsNil)
				})

				c.Specify("correctly processes timers", func() {
					sendTimer := func(vals ...int) {
						for _, v := range vals {
							statAccumInput.statChan <- Stat{"sample.timer", strconv.Itoa(int(v)),
								"ms", float32(1)}
						}
					}
					config.EmitInFields = true
					err := statAccumInput.Init(config)
					c.Assume(err, gs.IsNil)
					startInput()

					sendTimer(220, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100)

					close(statAccumInput.statChan)
					err = <-runErrChan
					c.Expect(err, gs.IsNil)

					msg := ith.Pack.Message

					getVal := func(token string) float64 {
						tmp, ok := msg.GetFieldValue("stats.timers.sample.timer." + token)
						c.Expect(ok, gs.IsTrue)
						val, ok := tmp.(float64)
						c.Expect(ok, gs.IsTrue)
						return val
					}

					c.Expect(getVal("upper"), gs.Equals, 220.0)
					c.Expect(getVal("lower"), gs.Equals, 10.0)
					c.Expect(getVal("mean"), gs.Equals, 70.0)
					c.Expect(getVal("upper_90"), gs.Equals, 100.0)
					c.Expect(getVal("mean_90"), gs.Equals, 55.0)
					tmp, ok := msg.GetFieldValue("stats.timers.sample.timer.count")
					c.Expect(ok, gs.IsTrue)
					intTmp, ok := tmp.(int64)
					c.Expect(ok, gs.IsTrue)
					c.Expect(intTmp, gs.Equals, int64(11))
				})
			})
		})
	})
}