func FilePollingInputSpec(c gs.Context) {
t := new(pipeline_ts.SimpleT)
ctrl := gomock.NewController(t)
tmpFileName := fmt.Sprintf("filepollinginput-test-%d", time.Now().UnixNano())
tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)
defer func() {
ctrl.Finish()
os.Remove(tmpFilePath)
}()
pConfig := NewPipelineConfig(nil)
var wg sync.WaitGroup
errChan := make(chan error, 1)
bytesChan := make(chan []byte, 1)
tickChan := make(chan time.Time)
retPackChan := make(chan *PipelinePack, 2)
defer close(retPackChan)
c.Specify("A FilePollingInput", func() {
input := new(FilePollingInput)
ith := new(plugins_ts.InputTestHelper)
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
config := input.ConfigStruct().(*FilePollingInputConfig)
config.FilePath = tmpFilePath
startInput := func(msgCount int) {
wg.Add(1)
go func() {
errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
wg.Done()
}()
}
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
c.Specify("gets updated information when reading a file", func() {
err := input.Init(config)
c.Assume(err, gs.IsNil)
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
nil).Return(io.EOF).Times(2)
splitCall.Do(func(f *os.File, del Deliverer) {
fBytes, err := ioutil.ReadAll(f)
if err != nil {
fBytes = []byte(err.Error())
}
bytesChan <- fBytes
})
startInput(2)
f, err := os.Create(tmpFilePath)
c.Expect(err, gs.IsNil)
_, err = f.Write([]byte("test1"))
c.Expect(err, gs.IsNil)
c.Expect(f.Close(), gs.IsNil)
tickChan <- time.Now()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "test1")
f, err = os.Create(tmpFilePath)
c.Expect(err, gs.IsNil)
_, err = f.Write([]byte("test2"))
c.Expect(err, gs.IsNil)
c.Expect(f.Close(), gs.IsNil)
tickChan <- time.Now()
msgBytes = <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "test2")
input.Stop()
wg.Wait()
c.Expect(<-errChan, gs.IsNil)
})
})
}
func LogstreamerInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
here, _ := os.Getwd()
dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")
tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
c.Expect(tmpErr, gs.Equals, nil)
defer func() {
tmpErr = os.RemoveAll(tmpDir)
c.Expect(tmpErr, gs.IsNil)
}()
globals := DefaultGlobals()
globals.BaseDir = tmpDir
pConfig := NewPipelineConfig(globals)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// Specify localhost, but we're not really going to use the network.
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
// Set up mock helper, runner, and pack supply channel.
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
c.Specify("A LogstreamerInput", func() {
lsInput := &LogstreamerInput{pConfig: pConfig}
lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
lsiConfig.LogDirectory = dirPath
lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
lsiConfig.Differentiator = []string{"logfile"}
lsiConfig.Priority = []string{"^Seq"}
c.Specify("w/ no translation map", func() {
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
c.Expect(len(lsInput.plugins), gs.Equals, 1)
// Create pool of packs.
numLines := 5 // # of lines in the log file we're parsing.
packs := make([]*PipelinePack, numLines)
ith.PackSupply = make(chan *PipelinePack, numLines)
for i := 0; i < numLines; i++ {
packs[i] = NewPipelinePack(ith.PackSupply)
ith.PackSupply <- packs[i]
}
c.Specify("reads a log file", func() {
// Expect InputRunner calls to get InChan and inject outgoing msgs.
ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
gomock.Any()).Times(numLines)
line := "boo hoo foo foo"
getRecCall.Return(len(line), []byte(line), nil)
getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
getRecCall.Return(0, make([]byte, 0), io.EOF)
deliverChan := make(chan []byte, 1)
deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
ith.MockDeliverer).Times(numLines)
deliverCall.Do(func(record []byte, del Deliverer) {
deliverChan <- record
})
ith.MockDeliverer.EXPECT().Done()
runOutChan := make(chan error, 1)
go func() {
err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
runOutChan <- err
}()
dur, _ := time.ParseDuration("5s")
timeout := time.After(dur)
timed := false
for x := 0; x < numLines; x++ {
select {
case record := <-deliverChan:
c.Expect(string(record), gs.Equals, line)
case <-timeout:
timed = true
x += numLines
}
// Free up the scheduler while we wait for the log file lines
// to be processed.
runtime.Gosched()
}
lsInput.Stop()
c.Expect(timed, gs.Equals, false)
c.Expect(<-runOutChan, gs.Equals, nil)
})
})
c.Specify("with a translation map", func() {
lsiConfig.Translation = make(ls.SubmatchTranslationMap)
lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)
c.Specify("allows len 1 translation map for 'missing'", func() {
lsiConfig.Translation["Seq"]["missing"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.IsNil)
})
c.Specify("doesn't allow len 1 map for other keys", func() {
lsiConfig.Translation["Seq"]["missin"] = 9999
err := lsInput.Init(lsiConfig)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"A translation map with one entry ('Seq') must be specifying a "+
"'missing' key.")
})
})
})
}
func ProcessDirectoryInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.PackSupply <- ith.Pack
err := pConfig.RegisterDefault("NullSplitter")
c.Assume(err, gs.IsNil)
c.Specify("A ProcessDirectoryInput", func() {
pdiInput := ProcessDirectoryInput{}
pdiInput.SetPipelineConfig(pConfig)
config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig)
workingDir, err := os.Getwd()
c.Assume(err, gs.IsNil)
config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes")
// `Ticker` is the last thing called during the setup part of the
// input's `Run` method, so it triggers a waitgroup that tests can
// wait on when they need to ensure initialization has finished.
var started sync.WaitGroup
started.Add(1)
tickChan := make(chan time.Time, 1)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do(
func() {
started.Done()
})
// Similarly we use a waitgroup to signal when LogMessage has been
// called to know when reloads have completed. Warning: If you call
// expectLogMessage with a msg that is never passed to LogMessage and
// then you call loaded.Wait() then your test will hang and never
// complete.
var loaded sync.WaitGroup
expectLogMessage := func(msg string) {
loaded.Add(1)
ith.MockInputRunner.EXPECT().LogMessage(msg).Do(
func(msg string) {
loaded.Done()
})
}
// Same name => same content.
paths := []string{
filepath.Join(config.ProcessDir, "100", "h0.toml"),
filepath.Join(config.ProcessDir, "100", "h1.toml"),
filepath.Join(config.ProcessDir, "200", "h0.toml"),
filepath.Join(config.ProcessDir, "300", "h1.toml"),
}
copyFile := func(src, dest string) {
inFile, err := os.Open(src)
c.Assume(err, gs.IsNil)
outFile, err := os.Create(dest)
c.Assume(err, gs.IsNil)
_, err = io.Copy(outFile, inFile)
c.Assume(err, gs.IsNil)
inFile.Close()
outFile.Close()
}
err = pdiInput.Init(config)
c.Expect(err, gs.IsNil)
for _, p := range paths {
expectLogMessage("Added: " + p)
}
go pdiInput.Run(ith.MockInputRunner, ith.MockHelper)
defer func() {
pdiInput.Stop()
for _, entry := range pdiInput.inputs {
entry.ir.Input().Stop()
}
}()
started.Wait()
c.Specify("loads scheduled jobs", func() {
pathIndex := func(name string) (i int) {
var p string
for i, p = range paths {
if name == p {
return
}
}
return -1
}
for name, entry := range pdiInput.inputs {
i := pathIndex(name)
// Make sure each file path got registered.
c.Expect(i, gs.Not(gs.Equals), -1)
dirName := filepath.Base(filepath.Dir(name))
dirInt, err := strconv.Atoi(dirName)
c.Expect(err, gs.IsNil)
// And that the ticker interval was read correctly.
c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval)
}
})
c.Specify("discovers and adds a new job", func() {
// Copy one of the files to register a new process.
newPath := filepath.Join(config.ProcessDir, "300", "h0.toml")
copyFile(paths[0], newPath)
defer func() {
err := os.Remove(newPath)
c.Assume(err, gs.IsNil)
}()
// Set up expectations and trigger process dir reload.
expectLogMessage("Added: " + newPath)
tickChan <- time.Now()
loaded.Wait()
// Make sure our plugin was loaded.
c.Expect(len(pdiInput.inputs), gs.Equals, 5)
newEntry, ok := pdiInput.inputs[newPath]
c.Expect(ok, gs.IsTrue)
c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300))
})
c.Specify("removes a deleted job", func() {
err := os.Remove(paths[3])
c.Assume(err, gs.IsNil)
defer func() {
copyFile(paths[1], paths[3])
}()
// Set up expectations and trigger process dir reload.
expectLogMessage("Removed: " + paths[3])
tickChan <- time.Now()
loaded.Wait()
// Make sure our plugin was deleted.
c.Expect(len(pdiInput.inputs), gs.Equals, 3)
})
c.Specify("notices a changed job", func() {
// Overwrite one job w/ a slightly different one.
copyFile(paths[0], paths[3])
defer copyFile(paths[1], paths[3])
// Set up expectations and trigger process dir reload.
expectLogMessage("Removed: " + paths[3])
expectLogMessage("Added: " + paths[3])
tickChan <- time.Now()
loaded.Wait()
// Make sure the new config was loaded.
c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals,
"hello world\n")
})
})
}
func TestReceivePayloadMessage(t *testing.T) {
b1 := sarama.NewMockBroker(t, 1)
b2 := sarama.NewMockBroker(t, 2)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests")
if tmpErr != nil {
t.Errorf("Unable to create a temporary directory: %s", tmpErr)
}
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Errorf("Cleanup failed: %s", err)
}
ctrl.Finish()
}()
topic := "test"
mdr := new(sarama.MetadataResponse)
mdr.AddBroker(b2.Addr(), b2.BrokerID())
mdr.AddTopicPartition(topic, 0, 2)
b1.Returns(mdr)
or := new(sarama.OffsetResponse)
or.AddTopicPartition(topic, 0, 0)
b2.Returns(or)
fr := new(sarama.FetchResponse)
fr.AddMessage(topic, 0, nil, sarama.ByteEncoder([]byte{0x41, 0x42}), 0)
b2.Returns(fr)
pConfig := NewPipelineConfig(nil)
pConfig.Globals.BaseDir = tmpDir
ki := new(KafkaInput)
ki.SetName(topic)
ki.SetPipelineConfig(pConfig)
config := ki.ConfigStruct().(*KafkaInputConfig)
config.Addrs = append(config.Addrs, b1.Addr())
config.Topic = topic
ith := new(plugins_ts.InputTestHelper)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
err := ki.Init(config)
if err != nil {
t.Fatalf("%s", err)
}
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
decChan := make(chan func(*PipelinePack), 1)
decCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
decCall.Do(func(dec func(pack *PipelinePack)) {
decChan <- dec
})
bytesChan := make(chan []byte, 1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
errChan := make(chan error)
go func() {
errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper)
}()
recd := <-bytesChan
if string(recd) != "AB" {
t.Errorf("Invalid Payload Expected: AB received: %s", string(recd))
}
packDec := <-decChan
packDec(ith.Pack)
if ith.Pack.Message.GetType() != "heka.kafka" {
t.Errorf("Invalid Type %s", ith.Pack.Message.GetType())
}
// There is a hang on the consumer close with the mock broker
// closing the brokers before the consumer works around the issue
// and is good enough for this test.
b1.Close()
b2.Close()
ki.Stop()
err = <-errChan
if err != nil {
t.Fatal(err)
}
filename := filepath.Join(tmpDir, "kafka", "test.test.0.offset.bin")
if o, err := readCheckpoint(filename); err != nil {
t.Errorf("Could not read the checkpoint file: %s", filename)
} else {
if o != 1 {
t.Errorf("Incorrect offset Expected: 1 Received: %d", o)
}
}
}
func HttpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`
c.Specify("A HttpInput", func() {
httpInput := HttpInput{}
ith := new(plugins_ts.InputTestHelper)
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
runOutputChan := make(chan error, 1)
startInput := func() {
go func() {
runOutputChan <- httpInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
}
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
// These assume that every sub-spec starts the input.
config := httpInput.ConfigStruct().(*HttpInputConfig)
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
ith.MockHelper.EXPECT().Hostname().Return("hekatests.example.com")
// These assume that every sub-spec makes exactly one HTTP request.
ith.MockInputRunner.EXPECT().NewSplitterRunner("0").Return(ith.MockSplitterRunner)
getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
getRecCall.Return(len(json_post), []byte(json_post), io.EOF)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
decChan := make(chan func(*PipelinePack), 1)
packDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
packDecCall.Do(func(dec func(*PipelinePack)) {
decChan <- dec
})
ith.MockSplitterRunner.EXPECT().DeliverRecord([]byte(json_post), nil)
ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()
splitter := &TokenSplitter{} // not actually used
ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)
c.Specify("honors time ticker to flush", func() {
// Spin up a http server.
server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876)
c.Expect(err, gs.IsNil)
go server.Start("/")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9876/"
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
// Getting the decorator means we've made our HTTP request.
<-decChan
})
c.Specify("supports configuring HTTP Basic Authentication", func() {
// Spin up a http server which expects username "user" and password "password"
server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875)
c.Expect(err, gs.IsNil)
go server.Start("/BasicAuthTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9875/BasicAuthTest"
config.User = "******"
config.Password = "******"
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
dec := <-decChan
dec(ith.Pack)
// we expect a statuscode 200 (i.e. success)
statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
})
c.Specify("supports configuring a different HTTP method", func() {
// Spin up a http server which expects requests with method "POST"
server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874)
c.Expect(err, gs.IsNil)
go server.Start("/PostTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9874/PostTest"
config.Method = "POST"
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
dec := <-decChan
dec(ith.Pack)
// we expect a statuscode 200 (i.e. success)
statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
})
c.Specify("supports configuring HTTP headers", func() {
// Spin up a http server which expects requests with method "POST"
server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873)
c.Expect(err, gs.IsNil)
go server.Start("/HeadersTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9873/HeadersTest"
config.Headers = map[string]string{"Accept": "text/plain"}
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
startInput()
tickChan <- time.Now()
dec := <-decChan
dec(ith.Pack)
// we expect a statuscode 200 (i.e. success)
statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
c.Assume(ok, gs.IsTrue)
c.Expect(statusCode, gs.Equals, int64(200))
})
c.Specify("supports configuring a request body", func() {
// Spin up a http server that echoes back the request body
server, err := plugins_ts.NewHttpBodyServer("localhost", 9872)
c.Expect(err, gs.IsNil)
go server.Start("/BodyTest")
time.Sleep(10 * time.Millisecond)
config.Url = "http://localhost:9872/BodyTest"
config.Method = "POST"
config.Body = json_post
err = httpInput.Init(config)
c.Assume(err, gs.IsNil)
respBodyChan := make(chan []byte, 1)
getRecCall.Do(func(r io.Reader) {
respBody := make([]byte, len(config.Body))
n, err := r.Read(respBody)
c.Expect(n, gs.Equals, len(config.Body))
c.Expect(err, gs.Equals, io.EOF)
respBodyChan <- respBody
})
startInput()
tickChan <- time.Now()
respBody := <-respBodyChan
c.Expect(string(respBody), gs.Equals, json_post)
})
httpInput.Stop()
runOutput := <-runOutputChan
c.Expect(runOutput, gs.IsNil)
})
}
func TcpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
c.Specify("A TcpInput", func() {
tcpInput := &TcpInput{}
config := &TcpInputConfig{
Net: "tcp",
Address: ith.AddrStr,
}
bytesChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var srDoneWG sync.WaitGroup
startServer := func() {
srDoneWG.Add(1)
ith.MockInputRunner.EXPECT().Name().Return("mock_name")
ith.MockInputRunner.EXPECT().NewDeliverer(gomock.Any()).Return(ith.MockDeliverer)
ith.MockDeliverer.EXPECT().Done()
ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
ith.MockSplitterRunner.EXPECT().Done().Do(func() {
srDoneWG.Done()
})
// splitCall gets called twice. The first time it returns nil, the
// second time it returns io.EOF.
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
ith.MockDeliverer).AnyTimes()
splitCall.Do(func(conn net.Conn, del Deliverer) {
recd, _ := ioutil.ReadAll(conn)
bytesChan <- recd
splitCall.Return(io.EOF)
})
err := tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}
c.Specify("not using TLS", func() {
err := tcpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(tcpInput.listener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
c.Specify("accepts connections and passes them to the splitter", func() {
go startServer()
data := []byte("THIS IS THE DATA")
outConn, err := net.Dial("tcp", ith.AddrStr)
c.Assume(err, gs.IsNil)
_, err = outConn.Write(data)
c.Expect(err, gs.IsNil)
outConn.Close()
recd := <-bytesChan
c.Expect(err, gs.IsNil)
c.Expect(string(recd), gs.Equals, string(data))
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
})
c.Specify("using TLS", func() {
config.UseTls = true
c.Specify("fails to init w/ missing key or cert file", func() {
config.Tls = TlsConfig{}
err := tcpInput.Init(config)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"TLS config requires both cert_file and key_file value.")
})
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
}
c.Specify("accepts connections and passes them to the splitter", func() {
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go startServer()
data := []byte("This is a test.")
clientConfig := new(tls.Config)
clientConfig.InsecureSkipVerify = true
outConn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Assume(err, gs.IsNil)
outConn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
n, err := outConn.Write(data)
c.Expect(err, gs.IsNil)
c.Expect(n, gs.Equals, len(data))
outConn.Close()
recd := <-bytesChan
c.Expect(err, gs.IsNil)
c.Expect(string(recd), gs.Equals, string(data))
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
c.Specify("doesn't accept connections below specified min TLS version", func() {
config.Tls.MinVersion = "TLS12"
err := tcpInput.Init(config)
c.Expect(err, gs.IsNil)
go startServer()
clientConfig := &tls.Config{
InsecureSkipVerify: true,
MaxVersion: tls.VersionTLS11,
}
conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
c.Expect(conn, gs.IsNil)
c.Expect(err, gs.Not(gs.IsNil))
<-bytesChan
tcpInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
srDoneWG.Wait()
})
})
})
}
func UdpInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// set up mock helper, decoder set, and packSupply channel
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
c.Specify("A UdpInput", func() {
udpInput := UdpInput{}
config := &UdpInputConfig{}
mbytes, _ := proto.Marshal(ith.Msg)
header := &message.Header{}
header.SetMessageLength(uint32(len(mbytes)))
hbytes, _ := proto.Marshal(header)
buf := encodeMessage(hbytes, mbytes)
bytesChan := make(chan []byte, 1)
ith.MockInputRunner.EXPECT().Name().Return("mock_name")
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().GetRemainingData().AnyTimes()
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
nil).AnyTimes()
splitCall.Do(func(conn net.Conn, del Deliverer) {
recd := make([]byte, 65536)
n, _ := conn.Read(recd)
recd = recd[:n]
bytesChan <- recd
})
c.Specify("using a udp address", func() {
ith.AddrStr = "localhost:55565"
ith.ResolvedAddrStr = "127.0.0.1:55565"
config.Net = "udp"
config.Address = ith.AddrStr
err := udpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UDPConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
c.Specify("passes the connection to SplitStream", func() {
go udpInput.Run(ith.MockInputRunner, ith.MockHelper)
conn, err := net.Dial("udp", ith.AddrStr)
c.Assume(err, gs.IsNil)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
conn.Close()
recd := <-bytesChan
c.Expect(string(recd), gs.Equals, string(buf))
udpInput.Stop()
})
})
if runtime.GOOS != "windows" {
c.Specify("using a unix datagram socket", func() {
tmpDir, err := ioutil.TempDir("", "heka-socket")
c.Assume(err, gs.IsNil)
unixPath := filepath.Join(tmpDir, "unixgram-socket")
ith.AddrStr = unixPath
config.Net = "unixgram"
config.Address = ith.AddrStr
err = udpInput.Init(config)
c.Assume(err, gs.IsNil)
realListener := (udpInput.listener).(*net.UnixConn)
c.Expect(realListener.LocalAddr().String(), gs.Equals, unixPath)
c.Specify("passes the socket to SplitStream", func() {
go udpInput.Run(ith.MockInputRunner, ith.MockHelper)
unixAddr, err := net.ResolveUnixAddr("unixgram", unixPath)
c.Assume(err, gs.IsNil)
conn, err := net.DialUnix("unixgram", nil, unixAddr)
c.Assume(err, gs.IsNil)
_, err = conn.Write(buf)
c.Assume(err, gs.IsNil)
conn.Close()
recd := <-bytesChan
c.Expect(string(recd), gs.Equals, string(buf))
udpInput.Stop()
})
})
}
})
}
func AMQPPluginSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
config := NewPipelineConfig(nil)
// Our two user/conn waitgroups.
ug := new(sync.WaitGroup)
cg := new(sync.WaitGroup)
// Setup the mock channel.
mch := NewMockAMQPChannel(ctrl)
// Setup the mock amqpHub with the mock chan return.
aqh := NewMockAMQPConnectionHub(ctrl)
aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)
errChan := make(chan error, 1)
bytesChan := make(chan []byte, 1)
c.Specify("An amqp input", func() {
// Setup all the mock calls for Init.
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
mch.EXPECT().QueueDeclare("", false, true, false, false,
gomock.Any()).Return(amqp.Queue{}, nil)
mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
mch.EXPECT().Qos(2, 0, false).Return(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.Pack = NewPipelinePack(config.InputRecycleChan())
// Set up relevant mocks.
ith.MockHelper = NewMockPluginHelper(ctrl)
ith.MockInputRunner = NewMockInputRunner(ctrl)
ith.MockSplitterRunner = NewMockSplitterRunner(ctrl)
ith.PackSupply = make(chan *PipelinePack, 1)
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
amqpInput := new(AMQPInput)
amqpInput.amqpHub = aqh
config := amqpInput.ConfigStruct().(*AMQPInputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
config.QueueTTL = 300000
err := amqpInput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpInput.ch, gs.Equals, mch)
c.Specify("consumes a text message", func() {
// Create a channel to send data to the input. Drop a message on
// there and close the channel.
streamChan := make(chan amqp.Delivery, 1)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "text/plain",
Body: []byte("This is a message"),
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "This is a message")
close(streamChan)
err = <-errChan
})
c.Specify("consumes a protobuf encoded message", func() {
encoder := client.NewProtobufEncoder(nil)
streamChan := make(chan amqp.Delivery, 1)
msg := new(message.Message)
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("logfile")
msg.SetLogger("/a/nice/path")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(0)
msg.SetPayload("This is a message")
msg.SetHostname("TestHost")
msgBody := make([]byte, 0, 500)
_ = encoder.EncodeMessageStream(msg, &msgBody)
ack := plugins_ts.NewMockAcknowledger(ctrl)
ack.EXPECT().Ack(gomock.Any(), false)
streamChan <- amqp.Delivery{
ContentType: "application/hekad",
Body: msgBody,
Timestamp: time.Now(),
Acknowledger: ack,
}
mch.EXPECT().Consume("", "", false, false, false, false,
gomock.Any()).Return(streamChan, nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
go func() {
err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, string(msgBody))
close(streamChan)
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
c.Specify("An amqp output", func() {
oth := plugins_ts.NewOutputTestHelper(ctrl)
pConfig := NewPipelineConfig(nil)
amqpOutput := new(AMQPOutput)
amqpOutput.amqpHub = aqh
config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
config.URL = ""
config.Exchange = ""
config.ExchangeType = ""
config.RoutingKey = "test"
closeChan := make(chan *amqp.Error)
inChan := make(chan *PipelinePack, 1)
mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
gomock.Any()).Return(nil)
// Increase the usage since Run decrements it on close.
ug.Add(1)
// Expect the close and the InChan calls.
aqh.EXPECT().Close("", cg)
oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
msg := pipeline_ts.GetTestMessage()
pack := NewPipelinePack(pConfig.InputRecycleChan())
pack.Message = msg
pack.Decoded = true
c.Specify("publishes a plain message", func() {
encoder := new(plugins.PayloadEncoder)
econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
econfig.AppendNewlines = false
encoder.Init(econfig)
payloadBytes, err := encoder.Encode(pack)
config.Encoder = "PayloadEncoder"
config.ContentType = "text/plain"
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("publishes a serialized message", func() {
encoder := new(ProtobufEncoder)
encoder.SetPipelineConfig(pConfig)
encoder.Init(nil)
protoBytes, err := encoder.Encode(pack)
c.Expect(err, gs.IsNil)
oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)
err = amqpOutput.Init(config)
c.Assume(err, gs.IsNil)
c.Expect(amqpOutput.ch, gs.Equals, mch)
mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
inChan <- pack
close(inChan)
close(closeChan)
go func() {
err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
errChan <- err
}()
ug.Wait()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
}
func ProcessInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Msg = pipeline_ts.GetTestMessage()
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
c.Specify("A ProcessInput", func() {
pInput := ProcessInput{}
config := pInput.ConfigStruct().(*ProcessInputConfig)
config.Command = make(map[string]cmdConfig)
ith.MockHelper.EXPECT().Hostname().Return(pConfig.Hostname())
tickChan := make(chan time.Time)
ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
errChan := make(chan error)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
decChan := make(chan func(*PipelinePack), 1)
setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
setDecCall.Do(func(dec func(*PipelinePack)) {
decChan <- dec
})
bytesChan := make(chan []byte, 1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
ith.MockDeliverer).Return(nil)
splitCall.Do(func(r io.Reader, del Deliverer) {
bytes, err := ioutil.ReadAll(r)
c.Assume(err, gs.IsNil)
bytesChan <- bytes
})
ith.MockDeliverer.EXPECT().Done()
c.Specify("using stdout", func() {
ith.MockInputRunner.EXPECT().NewDeliverer("stdout").Return(ith.MockDeliverer)
ith.MockInputRunner.EXPECT().NewSplitterRunner("stdout").Return(
ith.MockSplitterRunner)
c.Specify("reads a message from ProcessInput", func() {
pInput.SetName("SimpleTest")
// Note that no working directory is explicitly specified.
config.Command["0"] = cmdConfig{
Bin: PROCESSINPUT_TEST1_CMD,
Args: PROCESSINPUT_TEST1_CMD_ARGS,
}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
actual := <-bytesChan
c.Expect(string(actual), gs.Equals, PROCESSINPUT_TEST1_OUTPUT+"\n")
dec := <-decChan
dec(ith.Pack)
fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")
pInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
})
c.Specify("can pipe multiple commands together", func() {
pInput.SetName("PipedCmd")
// Note that no working directory is explicitly specified.
config.Command["0"] = cmdConfig{
Bin: PROCESSINPUT_PIPE_CMD1,
Args: PROCESSINPUT_PIPE_CMD1_ARGS,
}
config.Command["1"] = cmdConfig{
Bin: PROCESSINPUT_PIPE_CMD2,
Args: PROCESSINPUT_PIPE_CMD2_ARGS,
}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
go func() {
errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
actual := <-bytesChan
c.Expect(string(actual), gs.Equals, PROCESSINPUT_PIPE_OUTPUT+"\n")
dec := <-decChan
dec(ith.Pack)
fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName")
c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")
pInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
c.Specify("using stderr", func() {
ith.MockInputRunner.EXPECT().NewDeliverer("stderr").Return(ith.MockDeliverer)
ith.MockInputRunner.EXPECT().NewSplitterRunner("stderr").Return(
ith.MockSplitterRunner)
c.Specify("handles bad arguments", func() {
pInput.SetName("BadArgs")
config.ParseStdout = false
config.ParseStderr = true
// Note that no working directory is explicitly specified.
config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS}
err := pInput.Init(config)
c.Assume(err, gs.IsNil)
expectedErr := fmt.Errorf(
"BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]")
ith.MockInputRunner.EXPECT().LogError(expectedErr)
go func() {
errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
}()
tickChan <- time.Now()
// Error message differs by platform, but we at least wait
// until we get it.
<-bytesChan
pInput.Stop()
err = <-errChan
c.Expect(err, gs.IsNil)
})
})
})
}
func HttpListenInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
httpListenInput := HttpListenInput{}
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
splitter := &TokenSplitter{} // Not actually used.
errChan := make(chan error, 1)
startInput := func() {
go func() {
err := httpListenInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
}
config := httpListenInput.ConfigStruct().(*HttpListenInputConfig)
config.Address = "127.0.0.1:58325"
c.Specify("A HttpListenInput", func() {
startedChan := make(chan bool, 1)
defer close(startedChan)
ts := httptest.NewUnstartedServer(nil)
httpListenInput.starterFunc = func(hli *HttpListenInput) error {
ts.Start()
startedChan <- true
return nil
}
// These EXPECTs imply that every spec below will send exactly one
// HTTP request to the input.
ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)
decChan := make(chan func(*PipelinePack), 1)
feedDecorator := func(decorator func(*PipelinePack)) {
decChan <- decorator
}
setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
setDecCall.Do(feedDecorator)
streamChan := make(chan io.Reader, 1)
feedStream := func(r io.Reader) {
streamChan <- r
}
getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
gomock.Any()).Do(feedStream)
bytesChan := make(chan []byte, 1)
deliver := func(msgBytes []byte, del Deliverer) {
bytesChan <- msgBytes
}
ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()
c.Specify("Adds query parameters to the message pack as fields", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
getRecCall.Return(0, make([]byte, 0), io.EOF)
startInput()
<-startedChan
resp, err := http.Get(ts.URL + "/?test=Hello%20World")
resp.Body.Close()
c.Assume(err, gs.IsNil)
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("test")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "Hello World")
})
c.Specify("Add custom headers", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
getRecCall.Return(0, make([]byte, 0), io.EOF)
startInput()
<-startedChan
resp, err := http.Get(ts.URL)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
// Verify headers are there
eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
c.Expect(eq, gs.IsTrue)
eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
c.Expect(eq, gs.IsTrue)
})
c.Specify("Request body is sent as record", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
body := "1+2"
getRecCall.Return(0, []byte(body), io.EOF)
startInput()
<-startedChan
deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
nil)
deliverCall.Do(deliver)
resp, err := http.Post(ts.URL, "text/plain", strings.NewReader(body))
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "1+2")
})
c.Specify("Add request headers as fields", func() {
config.RequestHeaders = []string{
"X-REQUEST-ID",
}
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
getRecCall.Return(0, make([]byte, 0), io.EOF)
startInput()
<-startedChan
client := &http.Client{}
req, err := http.NewRequest("GET", ts.URL, nil)
req.Header.Add("X-REQUEST-ID", "12345")
resp, err := client.Do(req)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("X-REQUEST-ID")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "12345")
})
c.Specify("Add other request properties as fields", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
getRecCall.Return(0, make([]byte, 0), io.EOF)
startInput()
<-startedChan
client := &http.Client{}
req, err := http.NewRequest("GET", ts.URL+"/foo/bar?baz=2", nil)
c.Assume(err, gs.IsNil)
req.Host = "incoming.example.com:8080"
resp, err := client.Do(req)
resp.Body.Close()
c.Assume(err, gs.IsNil)
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("Host")
c.Assume(ok, gs.IsTrue)
// Host should not include port number.
c.Expect(fieldValue, gs.Equals, "incoming.example.com")
fieldValue, ok = ith.Pack.Message.GetFieldValue("Path")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "/foo/bar")
hostname, err := os.Hostname()
c.Assume(err, gs.IsNil)
c.Expect(*ith.Pack.Message.Hostname, gs.Equals, hostname)
c.Expect(*ith.Pack.Message.EnvVersion, gs.Equals, "1")
fieldValue, ok = ith.Pack.Message.GetFieldValue("RemoteAddr")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue != nil, gs.IsTrue)
fieldValueStr, ok := fieldValue.(string)
c.Assume(ok, gs.IsTrue)
c.Expect(len(fieldValueStr) > 0, gs.IsTrue)
// Per the `Request` docs, this should be an IP address:
// http://golang.org/pkg/net/http/#Request
ip := net.ParseIP(fieldValueStr)
c.Expect(ip != nil, gs.IsTrue)
})
ts.Close()
httpListenInput.Stop()
err := <-errChan
c.Expect(err, gs.IsNil)
})
}
func TestReceiveProtobufMessage(t *testing.T) {
broker := sarama.NewMockBroker(t, 2)
ctrl := gomock.NewController(t)
tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests")
if tmpErr != nil {
t.Errorf("Unable to create a temporary directory: %s", tmpErr)
}
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
t.Errorf("Cleanup failed: %s", err)
}
ctrl.Finish()
}()
topic := "test"
mockFetchResponse := sarama.NewMockFetchResponse(t, 1)
mockFetchResponse.SetMessage(topic, 0, 0, sarama.ByteEncoder([]byte{0x41, 0x42}))
broker.SetHandlerByMap(map[string]sarama.MockResponse{
"MetadataRequest": sarama.NewMockMetadataResponse(t).
SetBroker(broker.Addr(), broker.BrokerID()).
SetLeader(topic, 0, broker.BrokerID()),
"OffsetRequest": sarama.NewMockOffsetResponse(t).
SetOffset(topic, 0, sarama.OffsetOldest, 0).
SetOffset(topic, 0, sarama.OffsetNewest, 2),
"FetchRequest": mockFetchResponse,
})
pConfig := NewPipelineConfig(nil)
pConfig.Globals.BaseDir = tmpDir
ki := new(KafkaInput)
ki.SetName(topic)
ki.SetPipelineConfig(pConfig)
config := ki.ConfigStruct().(*KafkaInputConfig)
config.Addrs = append(config.Addrs, broker.Addr())
config.Topic = topic
ith := new(plugins_ts.InputTestHelper)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
err := ki.Init(config)
if err != nil {
t.Fatalf("%s", err)
}
ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
ith.MockSplitterRunner.EXPECT().Done()
bytesChan := make(chan []byte, 1)
splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil)
splitCall.Do(func(recd []byte, del Deliverer) {
bytesChan <- recd
})
errChan := make(chan error)
go func() {
errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper)
}()
recd := <-bytesChan
if string(recd) != "AB" {
t.Errorf("Invalid MsgBytes Expected: AB received: %s", string(recd))
}
// There is a hang on the consumer close with the mock broker
// closing the brokers before the consumer works around the issue
// and is good enough for this test.
broker.Close()
ki.Stop()
err = <-errChan
if err != nil {
t.Fatal(err)
}
}
func HttpListenInputSpec(c gs.Context) {
t := &pipeline_ts.SimpleT{}
ctrl := gomock.NewController(t)
defer ctrl.Finish()
pConfig := NewPipelineConfig(nil)
ith := new(plugins_ts.InputTestHelper)
ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
httpListenInput := HttpListenInput{}
ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
errChan := make(chan error, 1)
startInput := func() {
go func() {
err := httpListenInput.Run(ith.MockInputRunner, ith.MockHelper)
errChan <- err
}()
}
config := httpListenInput.ConfigStruct().(*HttpListenInputConfig)
config.Address = "127.0.0.1:58325"
c.Specify("A HttpListenInput", func() {
startedChan := make(chan bool, 1)
defer close(startedChan)
ts := httptest.NewUnstartedServer(nil)
httpListenInput.starterFunc = func(hli *HttpListenInput) error {
if hli.conf.UseTls {
ts.StartTLS()
} else {
ts.Start()
}
startedChan <- true
return nil
}
// These EXPECTs imply that every spec below will send exactly one
// HTTP request to the input.
ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
ith.MockSplitterRunner)
ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
ith.MockSplitterRunner.EXPECT().Done()
decChan := make(chan func(*PipelinePack), 1)
feedDecorator := func(decorator func(*PipelinePack)) {
decChan <- decorator
}
setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
setDecCall.Do(feedDecorator)
splitCall := ith.MockSplitterRunner.EXPECT().SplitStreamNullSplitterToEOF(gomock.Any(),
nil)
bytesChan := make(chan []byte, 1)
splitAndDeliver := func(r io.Reader, del Deliverer) {
msgBytes, _ := ioutil.ReadAll(r)
bytesChan <- msgBytes
}
c.Specify("Adds query parameters to the message pack as fields", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
splitCall.Return(io.EOF)
startInput()
<-startedChan
resp, err := http.Get(ts.URL + "/?test=Hello%20World")
resp.Body.Close()
c.Assume(err, gs.IsNil)
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("test")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "Hello World")
})
c.Specify("Add custom headers", func() {
config.Headers = http.Header{
"One": []string{"two", "three"},
"Four": []string{"five", "six", "seven"},
}
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
splitCall.Return(io.EOF)
startInput()
<-startedChan
resp, err := http.Get(ts.URL)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
// Verify headers are there
eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
c.Expect(eq, gs.IsTrue)
eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
c.Expect(eq, gs.IsTrue)
})
c.Specify("Request body is sent as record", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
body := "1+2"
splitCall.Return(io.EOF)
splitCall.Do(splitAndDeliver)
startInput()
<-startedChan
resp, err := http.Post(ts.URL, "text/plain", strings.NewReader(body))
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
msgBytes := <-bytesChan
c.Expect(string(msgBytes), gs.Equals, "1+2")
})
c.Specify("Add request headers as fields", func() {
config.RequestHeaders = []string{
"X-REQUEST-ID",
}
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
splitCall.Return(io.EOF)
startInput()
<-startedChan
client := &http.Client{}
req, err := http.NewRequest("GET", ts.URL, nil)
req.Header.Add("X-REQUEST-ID", "12345")
resp, err := client.Do(req)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("X-REQUEST-ID")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "12345")
})
c.Specify("Add other request properties as fields", func() {
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
splitCall.Return(io.EOF)
startInput()
<-startedChan
client := &http.Client{}
req, err := http.NewRequest("GET", ts.URL+"/foo/bar?baz=2", nil)
c.Assume(err, gs.IsNil)
req.Host = "incoming.example.com:8080"
resp, err := client.Do(req)
resp.Body.Close()
c.Assume(err, gs.IsNil)
c.Assume(resp.StatusCode, gs.Equals, 200)
packDec := <-decChan
packDec(ith.Pack)
fieldValue, ok := ith.Pack.Message.GetFieldValue("Host")
c.Assume(ok, gs.IsTrue)
// Host should not include port number.
c.Expect(fieldValue, gs.Equals, "incoming.example.com")
fieldValue, ok = ith.Pack.Message.GetFieldValue("Path")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue, gs.Equals, "/foo/bar")
hostname, err := os.Hostname()
c.Assume(err, gs.IsNil)
c.Expect(*ith.Pack.Message.Hostname, gs.Equals, hostname)
c.Expect(*ith.Pack.Message.EnvVersion, gs.Equals, "1")
fieldValue, ok = ith.Pack.Message.GetFieldValue("RemoteAddr")
c.Assume(ok, gs.IsTrue)
c.Expect(fieldValue != nil, gs.IsTrue)
fieldValueStr, ok := fieldValue.(string)
c.Assume(ok, gs.IsTrue)
c.Expect(len(fieldValueStr) > 0, gs.IsTrue)
// Per the `Request` docs, this should be an IP address:
// http://golang.org/pkg/net/http/#Request
ip := net.ParseIP(fieldValueStr)
c.Expect(ip != nil, gs.IsTrue)
})
c.Specify("Test API Authentication", func() {
config.AuthType = "API"
config.Key = "123"
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
splitCall.Return(io.EOF)
startInput()
<-startedChan
client := &http.Client{}
req, err := http.NewRequest("GET", ts.URL, nil)
req.Header.Add("X-API-KEY", "123")
resp, err := client.Do(req)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Expect(resp.StatusCode, gs.Equals, 200)
})
c.Specify("Test Basic Auth", func() {
config.AuthType = "Basic"
config.Username = "******"
config.Password = "******"
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
splitCall.Return(io.EOF)
startInput()
<-startedChan
client := &http.Client{}
req, err := http.NewRequest("GET", ts.URL, nil)
req.SetBasicAuth("foo", "bar")
resp, err := client.Do(req)
c.Assume(err, gs.IsNil)
resp.Body.Close()
c.Expect(resp.StatusCode, gs.Equals, 200)
})
c.Specify("Test TLS", func() {
config.UseTls = true
c.Specify("fails to init w/ missing key or cert file", func() {
config.Tls = TlsConfig{}
err := httpListenInput.setupTls(&config.Tls)
c.Expect(err, gs.Not(gs.IsNil))
c.Expect(err.Error(), gs.Equals,
"TLS config requires both cert_file and key_file value.")
})
config.Tls = TlsConfig{
CertFile: "./testsupport/cert.pem",
KeyFile: "./testsupport/key.pem",
}
err := httpListenInput.Init(config)
c.Assume(err, gs.IsNil)
ts.Config = httpListenInput.server
splitCall.Return(io.EOF)
startInput()
<-startedChan
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
req, err := http.NewRequest("GET", ts.URL, nil)
resp, err := client.Do(req)
c.Assume(err, gs.IsNil)
c.Expect(resp.TLS, gs.Not(gs.IsNil))
c.Expect(resp.StatusCode, gs.Equals, 200)
resp.Body.Close()
})
ts.Close()
httpListenInput.Stop()
err := <-errChan
c.Expect(err, gs.IsNil)
})
}