func ProcessDirectoryInputSpec(c gs.Context) {

	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	ith.PackSupply <- ith.Pack

	err := pConfig.RegisterDefault("NullSplitter")
	c.Assume(err, gs.IsNil)

	c.Specify("A ProcessDirectoryInput", func() {
		pdiInput := ProcessDirectoryInput{}
		pdiInput.SetPipelineConfig(pConfig)

		config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig)
		workingDir, err := os.Getwd()
		c.Assume(err, gs.IsNil)
		config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes")

		// `Ticker` is the last thing called during the setup part of the
		// input's `Run` method, so it triggers a waitgroup that tests can
		// wait on when they need to ensure initialization has finished.
		var started sync.WaitGroup
		started.Add(1)
		tickChan := make(chan time.Time, 1)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do(
			func() {
				started.Done()
			})

		// Similarly we use a waitgroup to signal when LogMessage has been
		// called to know when reloads have completed. Warning: If you call
		// expectLogMessage with a msg that is never passed to LogMessage and
		// then you call loaded.Wait() then your test will hang and never
		// complete.
		var loaded sync.WaitGroup
		expectLogMessage := func(msg string) {
			loaded.Add(1)
			ith.MockInputRunner.EXPECT().LogMessage(msg).Do(
				func(msg string) {
					loaded.Done()
				})
		}

		// Same name => same content.
		paths := []string{
			filepath.Join(config.ProcessDir, "100", "h0.toml"),
			filepath.Join(config.ProcessDir, "100", "h1.toml"),
			filepath.Join(config.ProcessDir, "200", "h0.toml"),
			filepath.Join(config.ProcessDir, "300", "h1.toml"),
		}

		copyFile := func(src, dest string) {
			inFile, err := os.Open(src)
			c.Assume(err, gs.IsNil)
			outFile, err := os.Create(dest)
			c.Assume(err, gs.IsNil)
			_, err = io.Copy(outFile, inFile)
			c.Assume(err, gs.IsNil)
			inFile.Close()
			outFile.Close()
		}

		err = pdiInput.Init(config)
		c.Expect(err, gs.IsNil)

		for _, p := range paths {
			expectLogMessage("Added: " + p)
		}
		go pdiInput.Run(ith.MockInputRunner, ith.MockHelper)
		defer func() {
			pdiInput.Stop()
			for _, entry := range pdiInput.inputs {
				entry.ir.Input().Stop()
			}
		}()
		started.Wait()

		c.Specify("loads scheduled jobs", func() {
			pathIndex := func(name string) (i int) {
				var p string
				for i, p = range paths {
					if name == p {
						return
					}
				}
				return -1
			}

			for name, entry := range pdiInput.inputs {
				i := pathIndex(name)
				// Make sure each file path got registered.
				c.Expect(i, gs.Not(gs.Equals), -1)
				dirName := filepath.Base(filepath.Dir(name))
				dirInt, err := strconv.Atoi(dirName)
				c.Expect(err, gs.IsNil)
				// And that the ticker interval was read correctly.
				c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval)
			}
		})

		c.Specify("discovers and adds a new job", func() {
			// Copy one of the files to register a new process.
			newPath := filepath.Join(config.ProcessDir, "300", "h0.toml")
			copyFile(paths[0], newPath)
			defer func() {
				err := os.Remove(newPath)
				c.Assume(err, gs.IsNil)
			}()

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Added: " + newPath)
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure our plugin was loaded.
			c.Expect(len(pdiInput.inputs), gs.Equals, 5)
			newEntry, ok := pdiInput.inputs[newPath]
			c.Expect(ok, gs.IsTrue)
			c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300))
		})

		c.Specify("removes a deleted job", func() {
			err := os.Remove(paths[3])
			c.Assume(err, gs.IsNil)
			defer func() {
				copyFile(paths[1], paths[3])
			}()

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Removed: " + paths[3])
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure our plugin was deleted.
			c.Expect(len(pdiInput.inputs), gs.Equals, 3)
		})

		c.Specify("notices a changed job", func() {
			// Overwrite one job w/ a slightly different one.
			copyFile(paths[0], paths[3])
			defer copyFile(paths[1], paths[3])

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Removed: " + paths[3])
			expectLogMessage("Added: " + paths[3])
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure the new config was loaded.
			c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals,
				"hello world\n")
		})
	})

}
Example #2
0
func FilePollingInputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	tmpFileName := fmt.Sprintf("filepollinginput-test-%d", time.Now().UnixNano())
	tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)

	defer func() {
		ctrl.Finish()
		os.Remove(tmpFilePath)
	}()

	pConfig := NewPipelineConfig(nil)
	var wg sync.WaitGroup
	errChan := make(chan error, 1)
	bytesChan := make(chan []byte, 1)
	tickChan := make(chan time.Time)

	retPackChan := make(chan *PipelinePack, 2)
	defer close(retPackChan)

	c.Specify("A FilePollingInput", func() {
		input := new(FilePollingInput)

		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
		ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

		config := input.ConfigStruct().(*FilePollingInputConfig)
		config.FilePath = tmpFilePath

		startInput := func(msgCount int) {
			wg.Add(1)
			go func() {
				errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
				wg.Done()
			}()
		}

		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
		ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

		c.Specify("gets updated information when reading a file", func() {

			err := input.Init(config)
			c.Assume(err, gs.IsNil)

			ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
				nil).Return(io.EOF).Times(2)
			splitCall.Do(func(f *os.File, del Deliverer) {
				fBytes, err := ioutil.ReadAll(f)
				if err != nil {
					fBytes = []byte(err.Error())
				}
				bytesChan <- fBytes
			})

			startInput(2)

			f, err := os.Create(tmpFilePath)
			c.Expect(err, gs.IsNil)

			_, err = f.Write([]byte("test1"))
			c.Expect(err, gs.IsNil)
			c.Expect(f.Close(), gs.IsNil)

			tickChan <- time.Now()

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "test1")

			f, err = os.Create(tmpFilePath)
			c.Expect(err, gs.IsNil)

			_, err = f.Write([]byte("test2"))
			c.Expect(err, gs.IsNil)
			c.Expect(f.Close(), gs.IsNil)

			tickChan <- time.Now()
			msgBytes = <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "test2")

			input.Stop()
			wg.Wait()
			c.Expect(<-errChan, gs.IsNil)
		})

	})

}
Example #3
0
func HttpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)

	json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`

	c.Specify("A HttpInput", func() {

		httpInput := HttpInput{}
		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
		ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

		runOutputChan := make(chan error, 1)
		startInput := func() {
			go func() {
				runOutputChan <- httpInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
		}

		ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

		// These assume that every sub-spec starts the input.
		config := httpInput.ConfigStruct().(*HttpInputConfig)
		tickChan := make(chan time.Time)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
		ith.MockHelper.EXPECT().Hostname().Return("hekatests.example.com")

		// These assume that every sub-spec makes exactly one HTTP request.
		ith.MockInputRunner.EXPECT().NewSplitterRunner("0").Return(ith.MockSplitterRunner)
		getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
		getRecCall.Return(len(json_post), []byte(json_post), io.EOF)
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		decChan := make(chan func(*PipelinePack), 1)
		packDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
		packDecCall.Do(func(dec func(*PipelinePack)) {
			decChan <- dec
		})
		ith.MockSplitterRunner.EXPECT().DeliverRecord([]byte(json_post), nil)
		ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()
		splitter := &TokenSplitter{} // not actually used
		ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)

		c.Specify("honors time ticker to flush", func() {
			// Spin up a http server.
			server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876)
			c.Expect(err, gs.IsNil)
			go server.Start("/")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9876/"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			// Getting the decorator means we've made our HTTP request.
			<-decChan
		})

		c.Specify("supports configuring HTTP Basic Authentication", func() {
			// Spin up a http server which expects username "user" and password "password"
			server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875)
			c.Expect(err, gs.IsNil)
			go server.Start("/BasicAuthTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9875/BasicAuthTest"
			config.User = "******"
			config.Password = "******"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring a different HTTP method", func() {
			// Spin up a http server which expects requests with method "POST"
			server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874)
			c.Expect(err, gs.IsNil)
			go server.Start("/PostTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9874/PostTest"
			config.Method = "POST"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring HTTP headers", func() {
			// Spin up a http server which expects requests with method "POST"
			server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873)
			c.Expect(err, gs.IsNil)
			go server.Start("/HeadersTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9873/HeadersTest"
			config.Headers = map[string]string{"Accept": "text/plain"}

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring a request body", func() {
			// Spin up a http server that echoes back the request body
			server, err := plugins_ts.NewHttpBodyServer("localhost", 9872)
			c.Expect(err, gs.IsNil)
			go server.Start("/BodyTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9872/BodyTest"
			config.Method = "POST"
			config.Body = json_post

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			respBodyChan := make(chan []byte, 1)
			getRecCall.Do(func(r io.Reader) {
				respBody := make([]byte, len(config.Body))
				n, err := r.Read(respBody)
				c.Expect(n, gs.Equals, len(config.Body))
				c.Expect(err, gs.Equals, io.EOF)
				respBodyChan <- respBody
			})

			startInput()
			tickChan <- time.Now()

			respBody := <-respBodyChan
			c.Expect(string(respBody), gs.Equals, json_post)
		})

		httpInput.Stop()
		runOutput := <-runOutputChan
		c.Expect(runOutput, gs.IsNil)
	})
}
Example #4
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	pConfig := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network.
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// Set up mock helper, runner, and pack supply channel.
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: pConfig}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs.
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
				ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
					ith.MockSplitterRunner)
				ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
				ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
				ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

				getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
					gomock.Any()).Times(numLines)
				line := "boo hoo foo foo"
				getRecCall.Return(len(line), []byte(line), nil)
				getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
				getRecCall.Return(0, make([]byte, 0), io.EOF)

				deliverChan := make(chan []byte, 1)
				deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
					ith.MockDeliverer).Times(numLines)
				deliverCall.Do(func(record []byte, del Deliverer) {
					deliverChan <- record
				})

				ith.MockDeliverer.EXPECT().Done()

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				dur, _ := time.ParseDuration("5s")
				timeout := time.After(dur)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case record := <-deliverChan:
						c.Expect(string(record), gs.Equals, line)
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Example #5
0
func UdpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	c.Specify("A UdpInput", func() {
		udpInput := UdpInput{}
		config := &UdpInputConfig{}

		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))
		hbytes, _ := proto.Marshal(header)
		buf := encodeMessage(hbytes, mbytes)

		bytesChan := make(chan []byte, 1)

		ith.MockInputRunner.EXPECT().Name().Return("mock_name")
		ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
		ith.MockSplitterRunner.EXPECT().GetRemainingData().AnyTimes()
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

		splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
			nil).AnyTimes()
		splitCall.Do(func(conn net.Conn, del Deliverer) {
			recd := make([]byte, 65536)
			n, _ := conn.Read(recd)
			recd = recd[:n]
			bytesChan <- recd
		})

		c.Specify("using a udp address", func() {
			ith.AddrStr = "localhost:55565"
			ith.ResolvedAddrStr = "127.0.0.1:55565"
			config.Net = "udp"
			config.Address = ith.AddrStr

			err := udpInput.Init(config)
			c.Assume(err, gs.IsNil)
			realListener := (udpInput.listener).(*net.UDPConn)
			c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)

			c.Specify("passes the connection to SplitStream", func() {
				go udpInput.Run(ith.MockInputRunner, ith.MockHelper)

				conn, err := net.Dial("udp", ith.AddrStr)
				c.Assume(err, gs.IsNil)
				_, err = conn.Write(buf)
				c.Assume(err, gs.IsNil)
				conn.Close()

				recd := <-bytesChan
				c.Expect(string(recd), gs.Equals, string(buf))
				udpInput.Stop()
			})
		})

		if runtime.GOOS != "windows" {
			c.Specify("using a unix datagram socket", func() {
				tmpDir, err := ioutil.TempDir("", "heka-socket")
				c.Assume(err, gs.IsNil)
				unixPath := filepath.Join(tmpDir, "unixgram-socket")
				ith.AddrStr = unixPath
				config.Net = "unixgram"
				config.Address = ith.AddrStr

				err = udpInput.Init(config)
				c.Assume(err, gs.IsNil)
				realListener := (udpInput.listener).(*net.UnixConn)
				c.Expect(realListener.LocalAddr().String(), gs.Equals, unixPath)

				c.Specify("passes the socket to SplitStream", func() {
					go udpInput.Run(ith.MockInputRunner, ith.MockHelper)

					unixAddr, err := net.ResolveUnixAddr("unixgram", unixPath)
					c.Assume(err, gs.IsNil)
					conn, err := net.DialUnix("unixgram", nil, unixAddr)
					c.Assume(err, gs.IsNil)
					_, err = conn.Write(buf)
					c.Assume(err, gs.IsNil)
					conn.Close()

					recd := <-bytesChan
					c.Expect(string(recd), gs.Equals, string(buf))
					udpInput.Stop()
				})
			})
		}
	})
}
Example #6
0
func TestReceivePayloadMessage(t *testing.T) {
	b1 := sarama.NewMockBroker(t, 1)
	b2 := sarama.NewMockBroker(t, 2)
	ctrl := gomock.NewController(t)
	tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests")
	if tmpErr != nil {
		t.Errorf("Unable to create a temporary directory: %s", tmpErr)
	}

	defer func() {
		if err := os.RemoveAll(tmpDir); err != nil {
			t.Errorf("Cleanup failed: %s", err)
		}
		ctrl.Finish()
	}()

	topic := "test"
	mdr := new(sarama.MetadataResponse)
	mdr.AddBroker(b2.Addr(), b2.BrokerID())
	mdr.AddTopicPartition(topic, 0, 2)
	b1.Returns(mdr)

	or := new(sarama.OffsetResponse)
	or.AddTopicPartition(topic, 0, 0)
	b2.Returns(or)

	fr := new(sarama.FetchResponse)
	fr.AddMessage(topic, 0, nil, sarama.ByteEncoder([]byte{0x41, 0x42}), 0)
	b2.Returns(fr)

	pConfig := NewPipelineConfig(nil)
	pConfig.Globals.BaseDir = tmpDir
	ki := new(KafkaInput)
	ki.SetName(topic)
	ki.SetPipelineConfig(pConfig)
	config := ki.ConfigStruct().(*KafkaInputConfig)
	config.Addrs = append(config.Addrs, b1.Addr())
	config.Topic = topic

	ith := new(plugins_ts.InputTestHelper)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)

	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	err := ki.Init(config)
	if err != nil {
		t.Fatalf("%s", err)
	}

	ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
	ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)

	decChan := make(chan func(*PipelinePack), 1)
	decCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
	decCall.Do(func(dec func(pack *PipelinePack)) {
		decChan <- dec
	})

	bytesChan := make(chan []byte, 1)
	splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil)
	splitCall.Do(func(recd []byte, del Deliverer) {
		bytesChan <- recd
	})

	errChan := make(chan error)
	go func() {
		errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper)
	}()

	recd := <-bytesChan
	if string(recd) != "AB" {
		t.Errorf("Invalid Payload Expected: AB received: %s", string(recd))
	}

	packDec := <-decChan
	packDec(ith.Pack)
	if ith.Pack.Message.GetType() != "heka.kafka" {
		t.Errorf("Invalid Type %s", ith.Pack.Message.GetType())
	}

	// There is a hang on the consumer close with the mock broker
	// closing the brokers before the consumer works around the issue
	// and is good enough for this test.
	b1.Close()
	b2.Close()

	ki.Stop()
	err = <-errChan
	if err != nil {
		t.Fatal(err)
	}

	filename := filepath.Join(tmpDir, "kafka", "test.test.0.offset.bin")
	if o, err := readCheckpoint(filename); err != nil {
		t.Errorf("Could not read the checkpoint file: %s", filename)
	} else {
		if o != 1 {
			t.Errorf("Incorrect offset Expected: 1 Received: %d", o)
		}
	}
}
Example #7
0
func TcpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A TcpInput", func() {
		tcpInput := &TcpInput{}
		config := &TcpInputConfig{
			Net:     "tcp",
			Address: ith.AddrStr,
		}

		bytesChan := make(chan []byte, 1)
		errChan := make(chan error, 1)
		var srDoneWG sync.WaitGroup

		startServer := func() {
			srDoneWG.Add(1)
			ith.MockInputRunner.EXPECT().Name().Return("mock_name")
			ith.MockInputRunner.EXPECT().NewDeliverer(gomock.Any()).Return(ith.MockDeliverer)
			ith.MockDeliverer.EXPECT().Done()
			ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
				ith.MockSplitterRunner)
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			ith.MockSplitterRunner.EXPECT().Done().Do(func() {
				srDoneWG.Done()
			})

			// splitCall gets called twice. The first time it returns nil, the
			// second time it returns io.EOF.
			splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
				ith.MockDeliverer).AnyTimes()
			splitCall.Do(func(conn net.Conn, del Deliverer) {
				recd, _ := ioutil.ReadAll(conn)
				bytesChan <- recd
				splitCall.Return(io.EOF)
			})

			err := tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			errChan <- err
		}

		c.Specify("not using TLS", func() {
			err := tcpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(tcpInput.listener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)

			c.Specify("accepts connections and passes them to the splitter", func() {
				go startServer()
				data := []byte("THIS IS THE DATA")

				outConn, err := net.Dial("tcp", ith.AddrStr)
				c.Assume(err, gs.IsNil)
				_, err = outConn.Write(data)
				c.Expect(err, gs.IsNil)
				outConn.Close()

				recd := <-bytesChan

				c.Expect(err, gs.IsNil)
				c.Expect(string(recd), gs.Equals, string(data))

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})
		})

		c.Specify("using TLS", func() {
			config.UseTls = true

			c.Specify("fails to init w/ missing key or cert file", func() {
				config.Tls = TlsConfig{}
				err := tcpInput.Init(config)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"TLS config requires both cert_file and key_file value.")
			})

			config.Tls = TlsConfig{
				CertFile: "./testsupport/cert.pem",
				KeyFile:  "./testsupport/key.pem",
			}

			c.Specify("accepts connections and passes them to the splitter", func() {
				err := tcpInput.Init(config)
				c.Expect(err, gs.IsNil)

				go startServer()
				data := []byte("This is a test.")

				clientConfig := new(tls.Config)
				clientConfig.InsecureSkipVerify = true
				outConn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
				c.Assume(err, gs.IsNil)
				outConn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
				n, err := outConn.Write(data)
				c.Expect(err, gs.IsNil)
				c.Expect(n, gs.Equals, len(data))
				outConn.Close()

				recd := <-bytesChan

				c.Expect(err, gs.IsNil)
				c.Expect(string(recd), gs.Equals, string(data))

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})

			c.Specify("doesn't accept connections below specified min TLS version", func() {
				config.Tls.MinVersion = "TLS12"
				err := tcpInput.Init(config)
				c.Expect(err, gs.IsNil)

				go startServer()

				clientConfig := &tls.Config{
					InsecureSkipVerify: true,
					MaxVersion:         tls.VersionTLS11,
				}
				conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
				c.Expect(conn, gs.IsNil)
				c.Expect(err, gs.Not(gs.IsNil))

				<-bytesChan

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})
		})
	})
}
Example #8
0
func ProcessInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()

	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	c.Specify("A ProcessInput", func() {
		pInput := ProcessInput{}

		config := pInput.ConfigStruct().(*ProcessInputConfig)
		config.Command = make(map[string]cmdConfig)

		ith.MockHelper.EXPECT().Hostname().Return(pConfig.Hostname())

		tickChan := make(chan time.Time)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

		errChan := make(chan error)

		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)

		decChan := make(chan func(*PipelinePack), 1)
		setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
		setDecCall.Do(func(dec func(*PipelinePack)) {
			decChan <- dec
		})

		bytesChan := make(chan []byte, 1)
		splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
			ith.MockDeliverer).Return(nil)
		splitCall.Do(func(r io.Reader, del Deliverer) {
			bytes, err := ioutil.ReadAll(r)
			c.Assume(err, gs.IsNil)
			bytesChan <- bytes
		})

		ith.MockDeliverer.EXPECT().Done()

		c.Specify("using stdout", func() {
			ith.MockInputRunner.EXPECT().NewDeliverer("stdout").Return(ith.MockDeliverer)
			ith.MockInputRunner.EXPECT().NewSplitterRunner("stdout").Return(
				ith.MockSplitterRunner)

			c.Specify("reads a message from ProcessInput", func() {
				pInput.SetName("SimpleTest")

				// Note that no working directory is explicitly specified.
				config.Command["0"] = cmdConfig{
					Bin:  PROCESSINPUT_TEST1_CMD,
					Args: PROCESSINPUT_TEST1_CMD_ARGS,
				}
				err := pInput.Init(config)
				c.Assume(err, gs.IsNil)

				go func() {
					errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
				}()
				tickChan <- time.Now()

				actual := <-bytesChan
				c.Expect(string(actual), gs.Equals, PROCESSINPUT_TEST1_OUTPUT+"\n")

				dec := <-decChan
				dec(ith.Pack)
				fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName")
				c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")

				pInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})

			c.Specify("can pipe multiple commands together", func() {
				pInput.SetName("PipedCmd")

				// Note that no working directory is explicitly specified.
				config.Command["0"] = cmdConfig{
					Bin:  PROCESSINPUT_PIPE_CMD1,
					Args: PROCESSINPUT_PIPE_CMD1_ARGS,
				}
				config.Command["1"] = cmdConfig{
					Bin:  PROCESSINPUT_PIPE_CMD2,
					Args: PROCESSINPUT_PIPE_CMD2_ARGS,
				}
				err := pInput.Init(config)
				c.Assume(err, gs.IsNil)

				go func() {
					errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
				}()
				tickChan <- time.Now()

				actual := <-bytesChan
				c.Expect(string(actual), gs.Equals, PROCESSINPUT_PIPE_OUTPUT+"\n")

				dec := <-decChan
				dec(ith.Pack)
				fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName")
				c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")

				pInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("using stderr", func() {
			ith.MockInputRunner.EXPECT().NewDeliverer("stderr").Return(ith.MockDeliverer)
			ith.MockInputRunner.EXPECT().NewSplitterRunner("stderr").Return(
				ith.MockSplitterRunner)

			c.Specify("handles bad arguments", func() {
				pInput.SetName("BadArgs")
				config.ParseStdout = false
				config.ParseStderr = true

				// Note that no working directory is explicitly specified.
				config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS}

				err := pInput.Init(config)
				c.Assume(err, gs.IsNil)

				expectedErr := fmt.Errorf(
					"BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]")
				ith.MockInputRunner.EXPECT().LogError(expectedErr)

				go func() {
					errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
				}()
				tickChan <- time.Now()

				// Error message differs by platform, but we at least wait
				// until we get it.
				<-bytesChan

				pInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})
	})
}
Example #9
0
func HttpListenInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	httpListenInput := HttpListenInput{}
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	splitter := &TokenSplitter{} // Not actually used.

	errChan := make(chan error, 1)
	startInput := func() {
		go func() {
			err := httpListenInput.Run(ith.MockInputRunner, ith.MockHelper)
			errChan <- err
		}()
	}

	config := httpListenInput.ConfigStruct().(*HttpListenInputConfig)
	config.Address = "127.0.0.1:58325"

	c.Specify("A HttpListenInput", func() {
		startedChan := make(chan bool, 1)
		defer close(startedChan)
		ts := httptest.NewUnstartedServer(nil)

		httpListenInput.starterFunc = func(hli *HttpListenInput) error {
			ts.Start()
			startedChan <- true
			return nil
		}

		// These EXPECTs imply that every spec below will send exactly one
		// HTTP request to the input.
		ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
			ith.MockSplitterRunner)
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)

		decChan := make(chan func(*PipelinePack), 1)
		feedDecorator := func(decorator func(*PipelinePack)) {
			decChan <- decorator
		}
		setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
		setDecCall.Do(feedDecorator)

		streamChan := make(chan io.Reader, 1)
		feedStream := func(r io.Reader) {
			streamChan <- r
		}
		getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
			gomock.Any()).Do(feedStream)

		bytesChan := make(chan []byte, 1)
		deliver := func(msgBytes []byte, del Deliverer) {
			bytesChan <- msgBytes
		}

		ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()

		c.Specify("Adds query parameters to the message pack as fields", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			getRecCall.Return(0, make([]byte, 0), io.EOF)
			startInput()
			<-startedChan
			resp, err := http.Get(ts.URL + "/?test=Hello%20World")
			resp.Body.Close()
			c.Assume(err, gs.IsNil)
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			fieldValue, ok := ith.Pack.Message.GetFieldValue("test")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "Hello World")
		})

		c.Specify("Add custom headers", func() {
			config.Headers = http.Header{
				"One":  []string{"two", "three"},
				"Four": []string{"five", "six", "seven"},
			}
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			getRecCall.Return(0, make([]byte, 0), io.EOF)
			startInput()
			<-startedChan
			resp, err := http.Get(ts.URL)
			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			// Verify headers are there
			eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
			c.Expect(eq, gs.IsTrue)
			eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
			c.Expect(eq, gs.IsTrue)
		})

		c.Specify("Request body is sent as record", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			body := "1+2"
			getRecCall.Return(0, []byte(body), io.EOF)
			startInput()
			<-startedChan
			deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
				nil)
			deliverCall.Do(deliver)
			resp, err := http.Post(ts.URL, "text/plain", strings.NewReader(body))
			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "1+2")
		})

		c.Specify("Add request headers as fields", func() {
			config.RequestHeaders = []string{
				"X-REQUEST-ID",
			}
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			getRecCall.Return(0, make([]byte, 0), io.EOF)
			startInput()
			<-startedChan

			client := &http.Client{}
			req, err := http.NewRequest("GET", ts.URL, nil)
			req.Header.Add("X-REQUEST-ID", "12345")
			resp, err := client.Do(req)

			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			fieldValue, ok := ith.Pack.Message.GetFieldValue("X-REQUEST-ID")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "12345")
		})

		c.Specify("Add other request properties as fields", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			getRecCall.Return(0, make([]byte, 0), io.EOF)
			startInput()
			<-startedChan

			client := &http.Client{}
			req, err := http.NewRequest("GET", ts.URL+"/foo/bar?baz=2", nil)
			c.Assume(err, gs.IsNil)
			req.Host = "incoming.example.com:8080"
			resp, err := client.Do(req)
			resp.Body.Close()
			c.Assume(err, gs.IsNil)
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)

			fieldValue, ok := ith.Pack.Message.GetFieldValue("Host")
			c.Assume(ok, gs.IsTrue)
			// Host should not include port number.
			c.Expect(fieldValue, gs.Equals, "incoming.example.com")
			fieldValue, ok = ith.Pack.Message.GetFieldValue("Path")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "/foo/bar")

			hostname, err := os.Hostname()
			c.Assume(err, gs.IsNil)
			c.Expect(*ith.Pack.Message.Hostname, gs.Equals, hostname)
			c.Expect(*ith.Pack.Message.EnvVersion, gs.Equals, "1")

			fieldValue, ok = ith.Pack.Message.GetFieldValue("RemoteAddr")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue != nil, gs.IsTrue)
			fieldValueStr, ok := fieldValue.(string)
			c.Assume(ok, gs.IsTrue)
			c.Expect(len(fieldValueStr) > 0, gs.IsTrue)
			// Per the `Request` docs, this should be an IP address:
			// http://golang.org/pkg/net/http/#Request
			ip := net.ParseIP(fieldValueStr)
			c.Expect(ip != nil, gs.IsTrue)
		})

		ts.Close()
		httpListenInput.Stop()
		err := <-errChan
		c.Expect(err, gs.IsNil)
	})
}
Example #10
0
func TestReceiveProtobufMessage(t *testing.T) {
	broker := sarama.NewMockBroker(t, 2)
	ctrl := gomock.NewController(t)
	tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests")
	if tmpErr != nil {
		t.Errorf("Unable to create a temporary directory: %s", tmpErr)
	}

	defer func() {
		if err := os.RemoveAll(tmpDir); err != nil {
			t.Errorf("Cleanup failed: %s", err)
		}
		ctrl.Finish()
	}()

	topic := "test"
	mockFetchResponse := sarama.NewMockFetchResponse(t, 1)
	mockFetchResponse.SetMessage(topic, 0, 0, sarama.ByteEncoder([]byte{0x41, 0x42}))

	broker.SetHandlerByMap(map[string]sarama.MockResponse{
		"MetadataRequest": sarama.NewMockMetadataResponse(t).
			SetBroker(broker.Addr(), broker.BrokerID()).
			SetLeader(topic, 0, broker.BrokerID()),
		"OffsetRequest": sarama.NewMockOffsetResponse(t).
			SetOffset(topic, 0, sarama.OffsetOldest, 0).
			SetOffset(topic, 0, sarama.OffsetNewest, 2),
		"FetchRequest": mockFetchResponse,
	})

	pConfig := NewPipelineConfig(nil)
	pConfig.Globals.BaseDir = tmpDir
	ki := new(KafkaInput)
	ki.SetName(topic)
	ki.SetPipelineConfig(pConfig)
	config := ki.ConfigStruct().(*KafkaInputConfig)
	config.Addrs = append(config.Addrs, broker.Addr())
	config.Topic = topic

	ith := new(plugins_ts.InputTestHelper)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)

	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	err := ki.Init(config)
	if err != nil {
		t.Fatalf("%s", err)
	}

	ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
	ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
	ith.MockSplitterRunner.EXPECT().Done()

	bytesChan := make(chan []byte, 1)
	splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil)
	splitCall.Do(func(recd []byte, del Deliverer) {
		bytesChan <- recd
	})

	errChan := make(chan error)
	go func() {
		errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper)
	}()

	recd := <-bytesChan
	if string(recd) != "AB" {
		t.Errorf("Invalid MsgBytes Expected: AB received: %s", string(recd))
	}

	// There is a hang on the consumer close with the mock broker
	// closing the brokers before the consumer works around the issue
	// and is good enough for this test.
	broker.Close()

	ki.Stop()
	err = <-errChan
	if err != nil {
		t.Fatal(err)
	}
}
Example #11
0
func HttpListenInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	httpListenInput := HttpListenInput{}
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	errChan := make(chan error, 1)
	startInput := func() {
		go func() {
			err := httpListenInput.Run(ith.MockInputRunner, ith.MockHelper)
			errChan <- err
		}()
	}

	config := httpListenInput.ConfigStruct().(*HttpListenInputConfig)
	config.Address = "127.0.0.1:58325"

	c.Specify("A HttpListenInput", func() {
		startedChan := make(chan bool, 1)
		defer close(startedChan)
		ts := httptest.NewUnstartedServer(nil)

		httpListenInput.starterFunc = func(hli *HttpListenInput) error {
			if hli.conf.UseTls {
				ts.StartTLS()
			} else {
				ts.Start()
			}

			startedChan <- true
			return nil
		}

		// These EXPECTs imply that every spec below will send exactly one
		// HTTP request to the input.
		ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
			ith.MockSplitterRunner)
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		ith.MockSplitterRunner.EXPECT().Done()

		decChan := make(chan func(*PipelinePack), 1)
		feedDecorator := func(decorator func(*PipelinePack)) {
			decChan <- decorator
		}
		setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
		setDecCall.Do(feedDecorator)

		splitCall := ith.MockSplitterRunner.EXPECT().SplitStreamNullSplitterToEOF(gomock.Any(),
			nil)

		bytesChan := make(chan []byte, 1)
		splitAndDeliver := func(r io.Reader, del Deliverer) {
			msgBytes, _ := ioutil.ReadAll(r)
			bytesChan <- msgBytes
		}

		c.Specify("Adds query parameters to the message pack as fields", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			splitCall.Return(io.EOF)
			startInput()
			<-startedChan
			resp, err := http.Get(ts.URL + "/?test=Hello%20World")
			resp.Body.Close()
			c.Assume(err, gs.IsNil)
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			fieldValue, ok := ith.Pack.Message.GetFieldValue("test")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "Hello World")
		})

		c.Specify("Add custom headers", func() {
			config.Headers = http.Header{
				"One":  []string{"two", "three"},
				"Four": []string{"five", "six", "seven"},
			}
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			splitCall.Return(io.EOF)
			startInput()
			<-startedChan
			resp, err := http.Get(ts.URL)
			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			// Verify headers are there
			eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
			c.Expect(eq, gs.IsTrue)
			eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
			c.Expect(eq, gs.IsTrue)
		})

		c.Specify("Request body is sent as record", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			body := "1+2"
			splitCall.Return(io.EOF)
			splitCall.Do(splitAndDeliver)
			startInput()
			<-startedChan
			resp, err := http.Post(ts.URL, "text/plain", strings.NewReader(body))
			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "1+2")
		})

		c.Specify("Add request headers as fields", func() {
			config.RequestHeaders = []string{
				"X-REQUEST-ID",
			}
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			splitCall.Return(io.EOF)
			startInput()
			<-startedChan

			client := &http.Client{}
			req, err := http.NewRequest("GET", ts.URL, nil)
			req.Header.Add("X-REQUEST-ID", "12345")
			resp, err := client.Do(req)

			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			fieldValue, ok := ith.Pack.Message.GetFieldValue("X-REQUEST-ID")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "12345")
		})

		c.Specify("Add other request properties as fields", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			splitCall.Return(io.EOF)
			startInput()
			<-startedChan

			client := &http.Client{}
			req, err := http.NewRequest("GET", ts.URL+"/foo/bar?baz=2", nil)
			c.Assume(err, gs.IsNil)
			req.Host = "incoming.example.com:8080"
			resp, err := client.Do(req)
			resp.Body.Close()
			c.Assume(err, gs.IsNil)
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)

			fieldValue, ok := ith.Pack.Message.GetFieldValue("Host")
			c.Assume(ok, gs.IsTrue)
			// Host should not include port number.
			c.Expect(fieldValue, gs.Equals, "incoming.example.com")
			fieldValue, ok = ith.Pack.Message.GetFieldValue("Path")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "/foo/bar")

			hostname, err := os.Hostname()
			c.Assume(err, gs.IsNil)
			c.Expect(*ith.Pack.Message.Hostname, gs.Equals, hostname)
			c.Expect(*ith.Pack.Message.EnvVersion, gs.Equals, "1")

			fieldValue, ok = ith.Pack.Message.GetFieldValue("RemoteAddr")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue != nil, gs.IsTrue)
			fieldValueStr, ok := fieldValue.(string)
			c.Assume(ok, gs.IsTrue)
			c.Expect(len(fieldValueStr) > 0, gs.IsTrue)
			// Per the `Request` docs, this should be an IP address:
			// http://golang.org/pkg/net/http/#Request
			ip := net.ParseIP(fieldValueStr)
			c.Expect(ip != nil, gs.IsTrue)
		})

		c.Specify("Test API Authentication", func() {
			config.AuthType = "API"
			config.Key = "123"

			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			splitCall.Return(io.EOF)
			startInput()
			<-startedChan

			client := &http.Client{}
			req, err := http.NewRequest("GET", ts.URL, nil)
			req.Header.Add("X-API-KEY", "123")
			resp, err := client.Do(req)
			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Expect(resp.StatusCode, gs.Equals, 200)
		})

		c.Specify("Test Basic Auth", func() {
			config.AuthType = "Basic"
			config.Username = "******"
			config.Password = "******"

			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			splitCall.Return(io.EOF)
			startInput()
			<-startedChan

			client := &http.Client{}
			req, err := http.NewRequest("GET", ts.URL, nil)
			req.SetBasicAuth("foo", "bar")
			resp, err := client.Do(req)
			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Expect(resp.StatusCode, gs.Equals, 200)
		})

		c.Specify("Test TLS", func() {
			config.UseTls = true

			c.Specify("fails to init w/ missing key or cert file", func() {
				config.Tls = TlsConfig{}

				err := httpListenInput.setupTls(&config.Tls)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"TLS config requires both cert_file and key_file value.")
			})

			config.Tls = TlsConfig{
				CertFile: "./testsupport/cert.pem",
				KeyFile:  "./testsupport/key.pem",
			}

			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			splitCall.Return(io.EOF)
			startInput()
			<-startedChan

			tr := &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			}
			client := &http.Client{Transport: tr}
			req, err := http.NewRequest("GET", ts.URL, nil)
			resp, err := client.Do(req)
			c.Assume(err, gs.IsNil)
			c.Expect(resp.TLS, gs.Not(gs.IsNil))
			c.Expect(resp.StatusCode, gs.Equals, 200)
			resp.Body.Close()

		})

		ts.Close()
		httpListenInput.Stop()
		err := <-errChan
		c.Expect(err, gs.IsNil)

	})
}