Ejemplo n.º 1
0
func StatsdInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
	ith.PackSupply = make(chan *PipelinePack, 1)

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, input runner, and stat accumulator
	ith.MockHelper = NewMockPluginHelper(ctrl)
	ith.MockInputRunner = NewMockInputRunner(ctrl)
	mockStatAccum := NewMockStatAccumulator(ctrl)

	c.Specify("A StatsdInput", func() {
		statsdInput := StatsdInput{}
		config := statsdInput.ConfigStruct().(*StatsdInputConfig)

		config.Address = ith.AddrStr
		err := statsdInput.Init(config)
		c.Assume(err, gs.IsNil)
		realListener := statsdInput.listener
		c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()
		mockListener := pipeline_ts.NewMockConn(ctrl)
		statsdInput.listener = mockListener

		ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil)
		mockListener.EXPECT().Close()
		mockListener.EXPECT().SetReadDeadline(gomock.Any())

		c.Specify("sends a Stat to the StatAccumulator", func() {
			statName := "sample.count"
			statVal := 303
			msg := fmt.Sprintf("%s:%d|c\n", statName, statVal)
			expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)}
			mockStatAccum.EXPECT().DropStat(expected).Return(true)
			readCall := mockListener.EXPECT().Read(make([]byte, 512))
			readCall.Return(len(msg), nil)
			readCall.Do(func(msgBytes []byte) {
				copy(msgBytes, []byte(msg))
				statsdInput.Stop()
			})
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()
			wg.Wait()
		})
	})
}
Ejemplo n.º 2
0
func UdpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A UdpInput", func() {
		udpInput := UdpInput{}
		err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr,
			Decoder:    "ProtobufDecoder",
			ParserType: "message.proto"})
		c.Assume(err, gs.IsNil)
		realListener := (udpInput.listener).(*net.UDPConn)
		c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)

		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Name().Return("UdpInput")
		encCall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "UdpInput-ProtobufDecoder")
		encCall.Return(ith.Decoder, true)

		c.Specify("reads a message from the connection and passes it to the decoder", func() {
			hbytes, _ := proto.Marshal(header)
			go func() {
				udpInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
			c.Assume(err, gs.IsNil)
			buf := encodeMessage(hbytes, mbytes)
			_, err = conn.Write(buf)
			c.Assume(err, gs.IsNil)
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			udpInput.Stop()
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
			c.Expect(ith.Pack.Decoded, gs.IsFalse)
		})
	})

	c.Specify("A UdpInput Multiline input", func() {
		ith.AddrStr = "localhost:55566"
		ith.ResolvedAddrStr = "127.0.0.1:55566"
		udpInput := UdpInput{}
		err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr,
			Decoder:    "test",
			ParserType: "token"})
		c.Assume(err, gs.IsNil)
		realListener := (udpInput.listener).(*net.UDPConn)
		c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).Times(2)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(2)
		ith.MockInputRunner.EXPECT().Name().Return("UdpInput").AnyTimes()
		encCall := ith.MockHelper.EXPECT().DecoderRunner("test", "UdpInput-test")
		encCall.Return(ith.Decoder, true)

		c.Specify("reads two messages from a packet and passes them to the decoder", func() {
			go func() {
				udpInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
			c.Assume(err, gs.IsNil)
			_, err = conn.Write([]byte("message1\nmessage2\n"))
			c.Assume(err, gs.IsNil)
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message1\n")
			ith.PackSupply <- ith.Pack
			packRef = <-ith.DecodeChan
			c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message2\n")
			udpInput.Stop()
		})
	})
}
Ejemplo n.º 3
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	config := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: config}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}
		lsiConfig.Decoder = "decoder-name"

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)
			mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
				// Expect calls to get decoder and decode each message. Since the
				// decoding is a no-op, the message payload will be the log file
				// line, unchanged.
				pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
					"-"+lsiConfig.Decoder)
				pbcall.Return(mockDecoderRunner, true)

				decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
				decodeCall.Return(ith.DecodeChan)

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				d, _ := time.ParseDuration("5s")
				timeout := time.After(d)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case <-ith.DecodeChan:
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Ejemplo n.º 4
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	pConfig := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network.
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// Set up mock helper, runner, and pack supply channel.
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: pConfig}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs.
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
				ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
					ith.MockSplitterRunner)
				ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
				ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
				ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

				getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
					gomock.Any()).Times(numLines)
				line := "boo hoo foo foo"
				getRecCall.Return(len(line), []byte(line), nil)
				getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
				getRecCall.Return(0, make([]byte, 0), io.EOF)

				deliverChan := make(chan []byte, 1)
				deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
					ith.MockDeliverer).Times(numLines)
				deliverCall.Do(func(record []byte, del Deliverer) {
					deliverChan <- record
				})

				ith.MockDeliverer.EXPECT().Done()

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				dur, _ := time.ParseDuration("5s")
				timeout := time.After(dur)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case record := <-deliverChan:
						c.Expect(string(record), gs.Equals, line)
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Ejemplo n.º 5
0
func TcpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)
	key := "testkey"
	signers := map[string]Signer{"test_1": {key}}
	signer := "test"

	c.Specify("A TcpInput protobuf parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
		tcpInput := TcpInput{}
		err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
			Signers:    signers,
			Decoder:    "ProtobufDecoder",
			ParserType: "message.proto"})
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr)
		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		enccall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "TcpInput-123-ProtobufDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		cleanup := func() {
			mockListener.EXPECT().Close()
			tcpInput.Stop()
			tcpInput.wg.Wait()
		}

		c.Specify("reads a message from its connection", func() {
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
		})

		c.Specify("reads a MD5 signed message from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(md5.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-ith.DecodeChan:
				c.Expect(ith.Pack, gs.Equals, packRef)
				c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
				c.Expect(ith.Pack.Signer, gs.Equals, "test")
			case t := <-timeout:
				c.Expect(t, gs.IsNil)
			}
		})

		c.Specify("reads a SHA1 signed message from its connection", func() {
			header.SetHmacHashFunction(message.Header_SHA1)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(sha1.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-ith.DecodeChan:
				c.Expect(ith.Pack, gs.Equals, packRef)
				c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
				c.Expect(ith.Pack.Signer, gs.Equals, "test")
			case t := <-timeout:
				c.Expect(t, gs.IsNil)
			}
		})

		c.Specify("reads a signed message with an expired key from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(11)) // non-existent key version
			hm := hmac.New(md5.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-mockDecoderRunner.InChan():
				c.Expect(packRef, gs.IsNil)
			case t := <-timeout:
				c.Expect(t, gs.IsTrue)
			}
		})

		c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(md5.New, []byte(key))
			hm.Write([]byte("some bytes"))
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-mockDecoderRunner.InChan():
				c.Expect(packRef, gs.IsNil)
			case t := <-timeout:
				c.Expect(t, gs.IsTrue)
			}
		})
	})

	c.Specify("A TcpInput regexp parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")

		config := &TcpInputConfig{
			Net:        "tcp",
			Address:    ith.AddrStr,
			Decoder:    "RegexpDecoder",
			ParserType: "regexp",
		}

		tcpInput := TcpInput{}
		err := tcpInput.Init(config)
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
		mbytes := []byte("this is a test message\n")
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Name().Return("logger")
		enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "TcpInput-123-RegexpDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		c.Specify("reads a message from its connection", func() {
			readCall.Return(len(mbytes), nil)
			readCall.Do(getPayloadText(mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				mockListener.EXPECT().Close()
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes[:len(mbytes)-1]))
			c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
			c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
		})
	})

	c.Specify("A TcpInput token parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
		tcpInput := TcpInput{}
		err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
			Decoder:    "TokenDecoder",
			ParserType: "token",
			Delimiter:  "\n"})
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
		mbytes := []byte("this is a test message\n")
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Name().Return("logger")
		enccall := ith.MockHelper.EXPECT().DecoderRunner("TokenDecoder", "TcpInput-123-TokenDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		c.Specify("reads a message from its connection", func() {
			readCall.Return(len(mbytes), nil)
			readCall.Do(getPayloadText(mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				mockListener.EXPECT().Close()
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes))
			c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
			c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
		})
	})

	c.Specify("A TcpInput using TLS", func() {
		tcpInput := TcpInput{}
		config := &TcpInputConfig{
			Net:        "tcp",
			Address:    ith.AddrStr,
			ParserType: "token",
			UseTls:     true,
		}

		c.Specify("fails to init w/ missing key or cert file", func() {
			config.Tls = TlsConfig{}
			err := tcpInput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("accepts TLS client connections", func() {
			ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
			config.Tls = TlsConfig{
				CertFile: "./testsupport/cert.pem",
				KeyFile:  "./testsupport/key.pem",
			}
			err := tcpInput.Init(config)
			c.Expect(err, gs.IsNil)
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()

			clientConfig := new(tls.Config)
			clientConfig.InsecureSkipVerify = true
			conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
			c.Expect(err, gs.IsNil)
			defer conn.Close()
			conn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
			n, err := conn.Write([]byte("This is a test."))
			c.Expect(err, gs.IsNil)
			c.Expect(n, gs.Equals, len("This is a test."))
		})

		c.Specify("doesn't accept connections below specified min TLS version", func() {
			ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
			config.Tls = TlsConfig{
				CertFile:   "./testsupport/cert.pem",
				KeyFile:    "./testsupport/key.pem",
				MinVersion: "TLS12",
			}
			err := tcpInput.Init(config)
			c.Expect(err, gs.IsNil)
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				tcpInput.Stop()
				tcpInput.wg.Wait()
				time.Sleep(time.Duration(1000))
			}()

			clientConfig := &tls.Config{
				InsecureSkipVerify: true,
				MaxVersion:         tls.VersionTLS11,
			}
			conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
			c.Expect(conn, gs.IsNil)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
Ejemplo n.º 6
0
func ProcessInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A ProcessInput", func() {
		pInput := ProcessInput{}

		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes()
		ith.MockInputRunner.EXPECT().Name().Return("logger").AnyTimes()

		enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "logger-RegexpDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).AnyTimes()

		config := pInput.ConfigStruct().(*ProcessInputConfig)
		config.Command = make(map[string]cmdConfig)

		pConfig := NewPipelineConfig(nil)
		ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

		tickChan := make(chan time.Time)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

		c.Specify("reads a message from ProcessInput", func() {

			pInput.SetName("SimpleTest")
			config.Decoder = "RegexpDecoder"
			config.ParserType = "token"
			config.Delimiter = "|"

			// Note that no working directory is explicitly specified
			config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_TEST1_CMD, Args: PROCESSINPUT_TEST1_CMD_ARGS}
			err := pInput.Init(config)
			c.Assume(err, gs.IsNil)

			go func() {
				pInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			tickChan <- time.Now()

			expected_payloads := PROCESSINPUT_TEST1_OUTPUT
			actual_payloads := []string{}

			for x := 0; x < 4; x++ {
				ith.PackSupply <- ith.Pack
				packRef := <-ith.DecodeChan
				c.Expect(ith.Pack, gs.Equals, packRef)
				actual_payloads = append(actual_payloads, *packRef.Message.Payload)
				fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
				c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")
				// Free up the scheduler
				runtime.Gosched()
			}

			for x := 0; x < 4; x++ {
				c.Expect(expected_payloads[x], gs.Equals, actual_payloads[x])
			}

			pInput.Stop()
		})

		c.Specify("handles bad arguments", func() {

			pInput.SetName("BadArgs")
			config.ParseStdout = false
			config.ParseStderr = true
			config.Decoder = "RegexpDecoder"
			config.ParserType = "token"
			config.Delimiter = "|"

			// Note that no working directory is explicitly specified
			config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS}

			err := pInput.Init(config)
			c.Assume(err, gs.IsNil)

			expected_err := fmt.Errorf("BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]")
			ith.MockInputRunner.EXPECT().LogError(expected_err)

			go func() {
				pInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			tickChan <- time.Now()

			ith.PackSupply <- ith.Pack
			<-ith.DecodeChan
			runtime.Gosched()

			pInput.Stop()
		})

		c.Specify("can pipe multiple commands together", func() {

			pInput.SetName("PipedCmd")
			config.Decoder = "RegexpDecoder"
			config.ParserType = "token"
			// Overload the delimiter
			config.Delimiter = " "

			// Note that no working directory is explicitly specified
			config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD1, Args: PROCESSINPUT_PIPE_CMD1_ARGS}
			config.Command["1"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD2, Args: PROCESSINPUT_PIPE_CMD2_ARGS}
			err := pInput.Init(config)
			c.Assume(err, gs.IsNil)

			go func() {
				pInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			tickChan <- time.Now()

			expected_payloads := PROCESSINPUT_PIPE_OUTPUT
			actual_payloads := []string{}

			for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ {
				ith.PackSupply <- ith.Pack
				packRef := <-ith.DecodeChan
				c.Expect(ith.Pack, gs.Equals, packRef)
				actual_payloads = append(actual_payloads, *packRef.Message.Payload)
				fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
				c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")
				// Free up the scheduler
				runtime.Gosched()
			}

			for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ {
				c.Expect(fmt.Sprintf("[%d] [%s] [%x]",
					len(actual_payloads[x]),
					actual_payloads[x],
					actual_payloads[x]),
					gs.Equals,
					fmt.Sprintf("[%d] [%s] [%x]",
						len(expected_payloads[x]),
						expected_payloads[x],
						expected_payloads[x]))
			}

			pInput.Stop()
		})

	})
}
Ejemplo n.º 7
0
func UdpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	c.Specify("A UdpInput", func() {
		udpInput := UdpInput{}
		config := &UdpInputConfig{}

		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))
		hbytes, _ := proto.Marshal(header)
		buf := encodeMessage(hbytes, mbytes)

		bytesChan := make(chan []byte, 1)

		ith.MockInputRunner.EXPECT().Name().Return("mock_name")
		ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
		ith.MockSplitterRunner.EXPECT().GetRemainingData().AnyTimes()
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

		splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
			nil).AnyTimes()
		splitCall.Do(func(conn net.Conn, del Deliverer) {
			recd := make([]byte, 65536)
			n, _ := conn.Read(recd)
			recd = recd[:n]
			bytesChan <- recd
		})

		c.Specify("using a udp address", func() {
			ith.AddrStr = "localhost:55565"
			ith.ResolvedAddrStr = "127.0.0.1:55565"
			config.Net = "udp"
			config.Address = ith.AddrStr

			err := udpInput.Init(config)
			c.Assume(err, gs.IsNil)
			realListener := (udpInput.listener).(*net.UDPConn)
			c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)

			c.Specify("passes the connection to SplitStream", func() {
				go udpInput.Run(ith.MockInputRunner, ith.MockHelper)

				conn, err := net.Dial("udp", ith.AddrStr)
				c.Assume(err, gs.IsNil)
				_, err = conn.Write(buf)
				c.Assume(err, gs.IsNil)
				conn.Close()

				recd := <-bytesChan
				c.Expect(string(recd), gs.Equals, string(buf))
				udpInput.Stop()
			})
		})

		if runtime.GOOS != "windows" {
			c.Specify("using a unix datagram socket", func() {
				tmpDir, err := ioutil.TempDir("", "heka-socket")
				c.Assume(err, gs.IsNil)
				unixPath := filepath.Join(tmpDir, "unixgram-socket")
				ith.AddrStr = unixPath
				config.Net = "unixgram"
				config.Address = ith.AddrStr

				err = udpInput.Init(config)
				c.Assume(err, gs.IsNil)
				realListener := (udpInput.listener).(*net.UnixConn)
				c.Expect(realListener.LocalAddr().String(), gs.Equals, unixPath)

				c.Specify("passes the socket to SplitStream", func() {
					go udpInput.Run(ith.MockInputRunner, ith.MockHelper)

					unixAddr, err := net.ResolveUnixAddr("unixgram", unixPath)
					c.Assume(err, gs.IsNil)
					conn, err := net.DialUnix("unixgram", nil, unixAddr)
					c.Assume(err, gs.IsNil)
					_, err = conn.Write(buf)
					c.Assume(err, gs.IsNil)
					conn.Close()

					recd := <-bytesChan
					c.Expect(string(recd), gs.Equals, string(buf))
					udpInput.Stop()
				})
			})
		}
	})
}
Ejemplo n.º 8
0
func TcpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A TcpInput", func() {
		tcpInput := &TcpInput{}
		config := &TcpInputConfig{
			Net:     "tcp",
			Address: ith.AddrStr,
		}

		bytesChan := make(chan []byte, 1)
		errChan := make(chan error, 1)
		var srDoneWG sync.WaitGroup

		startServer := func() {
			srDoneWG.Add(1)
			ith.MockInputRunner.EXPECT().Name().Return("mock_name")
			ith.MockInputRunner.EXPECT().NewDeliverer(gomock.Any()).Return(ith.MockDeliverer)
			ith.MockDeliverer.EXPECT().Done()
			ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
				ith.MockSplitterRunner)
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			ith.MockSplitterRunner.EXPECT().Done().Do(func() {
				srDoneWG.Done()
			})

			// splitCall gets called twice. The first time it returns nil, the
			// second time it returns io.EOF.
			splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
				ith.MockDeliverer).AnyTimes()
			splitCall.Do(func(conn net.Conn, del Deliverer) {
				recd, _ := ioutil.ReadAll(conn)
				bytesChan <- recd
				splitCall.Return(io.EOF)
			})

			err := tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			errChan <- err
		}

		c.Specify("not using TLS", func() {
			err := tcpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(tcpInput.listener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)

			c.Specify("accepts connections and passes them to the splitter", func() {
				go startServer()
				data := []byte("THIS IS THE DATA")

				outConn, err := net.Dial("tcp", ith.AddrStr)
				c.Assume(err, gs.IsNil)
				_, err = outConn.Write(data)
				c.Expect(err, gs.IsNil)
				outConn.Close()

				recd := <-bytesChan

				c.Expect(err, gs.IsNil)
				c.Expect(string(recd), gs.Equals, string(data))

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})
		})

		c.Specify("using TLS", func() {
			config.UseTls = true

			c.Specify("fails to init w/ missing key or cert file", func() {
				config.Tls = TlsConfig{}
				err := tcpInput.Init(config)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"TLS config requires both cert_file and key_file value.")
			})

			config.Tls = TlsConfig{
				CertFile: "./testsupport/cert.pem",
				KeyFile:  "./testsupport/key.pem",
			}

			c.Specify("accepts connections and passes them to the splitter", func() {
				err := tcpInput.Init(config)
				c.Expect(err, gs.IsNil)

				go startServer()
				data := []byte("This is a test.")

				clientConfig := new(tls.Config)
				clientConfig.InsecureSkipVerify = true
				outConn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
				c.Assume(err, gs.IsNil)
				outConn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
				n, err := outConn.Write(data)
				c.Expect(err, gs.IsNil)
				c.Expect(n, gs.Equals, len(data))
				outConn.Close()

				recd := <-bytesChan

				c.Expect(err, gs.IsNil)
				c.Expect(string(recd), gs.Equals, string(data))

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})

			c.Specify("doesn't accept connections below specified min TLS version", func() {
				config.Tls.MinVersion = "TLS12"
				err := tcpInput.Init(config)
				c.Expect(err, gs.IsNil)

				go startServer()

				clientConfig := &tls.Config{
					InsecureSkipVerify: true,
					MaxVersion:         tls.VersionTLS11,
				}
				conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
				c.Expect(conn, gs.IsNil)
				c.Expect(err, gs.Not(gs.IsNil))

				<-bytesChan

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})
		})
	})
}
Ejemplo n.º 9
0
func LogfileInputSpec0(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A LogFileInput", func() {
		tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
		c.Expect(tmpErr, gs.Equals, nil)
		origBaseDir := Globals().BaseDir
		Globals().BaseDir = tmpDir
		defer func() {
			Globals().BaseDir = origBaseDir
			tmpErr = os.RemoveAll(tmpDir)
			c.Expect(tmpErr, gs.IsNil)
		}()
		lfInput := new(LogfileInput)
		lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
		lfiConfig.SeekJournalName = "test-seekjournal"
		lfiConfig.LogFile = "../testsupport/test-zeus.log"
		lfiConfig.Logger = "zeus"
		lfiConfig.UseSeekJournal = true
		lfiConfig.Decoder = "decoder-name"
		lfiConfig.DiscoverInterval = 1
		lfiConfig.StatInterval = 1
		err := lfInput.Init(lfiConfig)
		c.Expect(err, gs.IsNil)
		mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

		// Create pool of packs.
		numLines := 95 // # of lines in the log file we're parsing.
		packs := make([]*PipelinePack, numLines)
		ith.PackSupply = make(chan *PipelinePack, numLines)
		for i := 0; i < numLines; i++ {
			packs[i] = NewPipelinePack(ith.PackSupply)
			ith.PackSupply <- packs[i]
		}

		c.Specify("reads a log file", func() {
			// Expect InputRunner calls to get InChan and inject outgoing msgs
			ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
			// Expect calls to get decoder and decode each message. Since the
			// decoding is a no-op, the message payload will be the log file
			// line, unchanged.
			ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
			pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder)
			pbcall.Return(mockDecoderRunner, true)
			decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
			decodeCall.Return(ith.DecodeChan)
			go func() {
				err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
			}()
			for x := 0; x < numLines; x++ {
				_ = <-ith.DecodeChan
				// Free up the scheduler while we wait for the log file lines
				// to be processed.
				runtime.Gosched()
			}
			lfInput.Stop()

			fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile)
			c.Expect(err, gs.IsNil)
			fileStr := string(fileBytes)
			lines := strings.Split(fileStr, "\n")
			for i, line := range lines {
				if line == "" {
					continue
				}
				c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n")
				c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus")
			}

			// Wait for the file update to hit the disk; better suggestions are welcome
			runtime.Gosched()
			time.Sleep(time.Millisecond * 250)
			journalData := []byte(`{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_len":300,"last_start":28650,"seek":28950}`)
			journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
			c.Expect(err, gs.IsNil)
			c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
		})

		c.Specify("uses the filename as the default logger name", func() {
			lfInput := new(LogfileInput)
			lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
			lfiConfig.LogFile = "../testsupport/test-zeus.log"

			lfiConfig.DiscoverInterval = 1
			lfiConfig.StatInterval = 1
			err := lfInput.Init(lfiConfig)
			c.Expect(err, gs.Equals, nil)

			c.Expect(lfInput.Monitor.logger_ident,
				gs.Equals,
				lfiConfig.LogFile)
		})
	})

	c.Specify("A Regex LogFileInput", func() {
		tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
		c.Expect(tmpErr, gs.Equals, nil)
		origBaseDir := Globals().BaseDir
		Globals().BaseDir = tmpDir
		defer func() {
			Globals().BaseDir = origBaseDir
			tmpErr = os.RemoveAll(tmpDir)
			c.Expect(tmpErr, gs.Equals, nil)
		}()
		lfInput := new(LogfileInput)
		lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
		lfiConfig.SeekJournalName = "regex-seekjournal"
		lfiConfig.LogFile = "../testsupport/test-zeus.log"
		lfiConfig.Logger = "zeus"
		lfiConfig.ParserType = "regexp"
		lfiConfig.UseSeekJournal = true
		lfiConfig.Decoder = "decoder-name"
		lfiConfig.DiscoverInterval = 1
		lfiConfig.StatInterval = 1

		mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

		// Create pool of packs.
		numLines := 95 // # of lines in the log file we're parsing.
		packs := make([]*PipelinePack, numLines)
		ith.PackSupply = make(chan *PipelinePack, numLines)
		for i := 0; i < numLines; i++ {
			packs[i] = NewPipelinePack(ith.PackSupply)
			ith.PackSupply <- packs[i]
		}

		c.Specify("doesn't whack default RegexpParser delimiter", func() {
			err := lfInput.Init(lfiConfig)
			c.Expect(err, gs.IsNil)
			parser := lfInput.Monitor.parser.(*RegexpParser)
			buf := bytes.NewBuffer([]byte("split\nhere"))
			n, r, err := parser.Parse(buf)
			c.Expect(n, gs.Equals, len("split\n"))
			c.Expect(string(r), gs.Equals, "split")
			c.Expect(err, gs.IsNil)
		})

		c.Specify("keeps captures in the record text", func() {
			lfiConfig.Delimiter = "(\n)"
			err := lfInput.Init(lfiConfig)
			c.Expect(err, gs.IsNil)
			parser := lfInput.Monitor.parser.(*RegexpParser)
			buf := bytes.NewBuffer([]byte("split\nhere"))
			n, r, err := parser.Parse(buf)
			c.Expect(n, gs.Equals, len("split\n"))
			c.Expect(string(r), gs.Equals, "split\n")
			c.Expect(err, gs.IsNil)
		})

		c.Specify("reads a log file", func() {
			lfiConfig.Delimiter = "(\n)"
			err := lfInput.Init(lfiConfig)
			c.Expect(err, gs.IsNil)

			// Expect InputRunner calls to get InChan and inject outgoing msgs
			ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
			// Expect calls to get decoder and decode each message. Since the
			// decoding is a no-op, the message payload will be the log file
			// line, unchanged.
			ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
			pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder)
			pbcall.Return(mockDecoderRunner, true)
			decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
			decodeCall.Return(ith.DecodeChan)
			go func() {
				err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
			}()
			for x := 0; x < numLines; x++ {
				_ = <-ith.DecodeChan
				// Free up the scheduler while we wait for the log file lines
				// to be processed.
				runtime.Gosched()
			}
			lfInput.Stop()

			fileBytes, err := ioutil.ReadFile(lfiConfig.LogFile)
			c.Expect(err, gs.IsNil)
			fileStr := string(fileBytes)
			lines := strings.Split(fileStr, "\n")
			for i, line := range lines {
				if line == "" {
					continue
				}
				c.Expect(packs[i].Message.GetPayload(), gs.Equals, line+"\n")
				c.Expect(packs[i].Message.GetLogger(), gs.Equals, "zeus")
			}

			// Wait for the file update to hit the disk; better suggestions are welcome
			runtime.Gosched()
			time.Sleep(time.Millisecond * 250)
			journalData := []byte(`{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_len":300,"last_start":28650,"seek":28950}`)
			journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
			c.Expect(err, gs.IsNil)
			c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
		})
	})

	c.Specify("A Regex Multiline LogFileInput", func() {
		tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
		c.Expect(tmpErr, gs.Equals, nil)
		origBaseDir := Globals().BaseDir
		Globals().BaseDir = tmpDir
		defer func() {
			Globals().BaseDir = origBaseDir
			tmpErr = os.RemoveAll(tmpDir)
			c.Expect(tmpErr, gs.Equals, nil)
		}()
		lfInput := new(LogfileInput)
		lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
		lfiConfig.SeekJournalName = "multiline-seekjournal"
		lfiConfig.LogFile = "../testsupport/multiline.log"
		lfiConfig.Logger = "multiline"
		lfiConfig.ParserType = "regexp"
		lfiConfig.Delimiter = "\n(\\d{4}-\\d{2}-\\d{2})"
		lfiConfig.DelimiterLocation = "start"
		lfiConfig.UseSeekJournal = true
		lfiConfig.Decoder = "decoder-name"
		lfiConfig.DiscoverInterval = 1
		lfiConfig.StatInterval = 1
		err := lfInput.Init(lfiConfig)
		c.Expect(err, gs.IsNil)

		mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

		// Create pool of packs.
		numLines := 4 // # of lines in the log file we're parsing.
		packs := make([]*PipelinePack, numLines)
		ith.PackSupply = make(chan *PipelinePack, numLines)
		for i := 0; i < numLines; i++ {
			packs[i] = NewPipelinePack(ith.PackSupply)
			ith.PackSupply <- packs[i]
		}

		c.Specify("reads a log file", func() {
			// Expect InputRunner calls to get InChan and inject outgoing msgs
			ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
			// Expect calls to get decoder and decode each message. Since the
			// decoding is a no-op, the message payload will be the log file
			// line, unchanged.
			ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
			pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder)
			pbcall.Return(mockDecoderRunner, true)
			decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
			decodeCall.Return(ith.DecodeChan)
			go func() {
				err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
			}()
			for x := 0; x < numLines; x++ {
				_ = <-ith.DecodeChan
				// Free up the scheduler while we wait for the log file lines
				// to be processed.
				runtime.Gosched()
			}
			lfInput.Stop()

			lines := []string{
				"2012-07-13 18:48:01 debug    readSocket()",
				"2012-07-13 18:48:21 info     Processing queue id 3496 -> subm id 2817 from site ms",
				"2012-07-13 18:48:25 debug    line0\nline1\nline2",
				"2012-07-13 18:48:26 debug    readSocket()",
			}
			for i, line := range lines {
				c.Expect(packs[i].Message.GetPayload(), gs.Equals, line)
				c.Expect(packs[i].Message.GetLogger(), gs.Equals, "multiline")
			}

			// Wait for the file update to hit the disk; better suggestions are welcome
			runtime.Gosched()
			time.Sleep(time.Millisecond * 250)
			journalData := []byte(`{"last_hash":"39e4c3e6e9c88a794b3e7c91c155682c34cf1a4a","last_len":41,"last_start":172,"seek":214}`)
			journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
			c.Expect(err, gs.IsNil)
			c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
		})
	})

	c.Specify("A message.proto LogFileInput", func() {
		tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
		c.Expect(tmpErr, gs.Equals, nil)
		origBaseDir := Globals().BaseDir
		Globals().BaseDir = tmpDir
		defer func() {
			Globals().BaseDir = origBaseDir
			tmpErr = os.RemoveAll(tmpDir)
			c.Expect(tmpErr, gs.Equals, nil)
		}()
		lfInput := new(LogfileInput)
		lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
		lfiConfig.SeekJournalName = "protobuf-seekjournal"
		lfiConfig.LogFile = "../testsupport/protobuf.log"
		lfiConfig.ParserType = "message.proto"
		lfiConfig.UseSeekJournal = true
		lfiConfig.Decoder = "decoder-name"
		lfiConfig.DiscoverInterval = 1
		lfiConfig.StatInterval = 1
		err := lfInput.Init(lfiConfig)
		c.Expect(err, gs.IsNil)

		mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

		// Create pool of packs.
		numLines := 7 // # of lines in the log file we're parsing.
		packs := make([]*PipelinePack, numLines)
		ith.PackSupply = make(chan *PipelinePack, numLines)
		for i := 0; i < numLines; i++ {
			packs[i] = NewPipelinePack(ith.PackSupply)
			ith.PackSupply <- packs[i]
		}

		c.Specify("reads a log file", func() {
			// Expect InputRunner calls to get InChan and decode outgoing msgs
			ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
			ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
			pbcall := ith.MockHelper.EXPECT().DecoderRunner(lfiConfig.Decoder, "LogfileInput-"+lfiConfig.Decoder)
			pbcall.Return(mockDecoderRunner, true)
			decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
			decodeCall.Return(ith.DecodeChan)
			go func() {
				err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
			}()
			for x := 0; x < numLines; x++ {
				_ = <-ith.DecodeChan
				// Free up the scheduler while we wait for the log file lines
				// to be processed.
				runtime.Gosched()
			}
			lfInput.Stop()

			lines := []int{36230, 41368, 42310, 41343, 37171, 56727, 46492}
			for i, line := range lines {
				c.Expect(len(packs[i].MsgBytes), gs.Equals, line)
				err = proto.Unmarshal(packs[i].MsgBytes, packs[i].Message)
				c.Expect(err, gs.IsNil)
				c.Expect(packs[i].Message.GetType(), gs.Equals, "hekabench")
			}

			// Wait for the file update to hit the disk; better suggestions are welcome
			runtime.Gosched()
			time.Sleep(time.Millisecond * 250)
			journalData := []byte(`{"last_hash":"f67dc6bbbbb6a91b59e661b6170de50c96eab100","last_len":46499,"last_start":255191,"seek":301690}`)
			journalFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "seekjournals", lfiConfig.SeekJournalName))
			c.Expect(err, gs.IsNil)
			c.Expect(bytes.Compare(journalData, journalFile), gs.Equals, 0)
		})
	})

	c.Specify("A message.proto LogFileInput no decoder", func() {
		tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
		c.Expect(tmpErr, gs.Equals, nil)
		origBaseDir := Globals().BaseDir
		Globals().BaseDir = tmpDir
		defer func() {
			Globals().BaseDir = origBaseDir
			tmpErr = os.RemoveAll(tmpDir)
			c.Expect(tmpErr, gs.Equals, nil)
		}()
		lfInput := new(LogfileInput)
		lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
		lfiConfig.SeekJournalName = "protobuf-seekjournal"
		lfiConfig.LogFile = "../testsupport/protobuf.log"
		lfiConfig.ParserType = "message.proto"
		lfiConfig.UseSeekJournal = true
		lfiConfig.Decoder = ""
		lfiConfig.DiscoverInterval = 1
		lfiConfig.StatInterval = 1
		err := lfInput.Init(lfiConfig)
		c.Expect(err, gs.Not(gs.IsNil))
	})
}
Ejemplo n.º 10
0
func LogfileInputSpec1(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
	c.Expect(tmpErr, gs.Equals, nil)
	origBaseDir := Globals().BaseDir
	Globals().BaseDir = tmpDir
	defer func() {
		Globals().BaseDir = origBaseDir
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.Equals, nil)
	}()
	journalName := "test-seekjournal"
	journalDir := filepath.Join(tmpDir, "seekjournal")
	tmpErr = os.MkdirAll(journalDir, 0770)
	c.Expect(tmpErr, gs.Equals, nil)

	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A LogfileInput", func() {
		c.Specify("save the seek position of the last complete logline", func() {
			lfInput, lfiConfig := createIncompleteLogfileInput(journalName)

			// Initialize the input test helper
			err := lfInput.Init(lfiConfig)
			c.Expect(err, gs.IsNil)

			dName := "decoder-name"
			lfInput.decoderName = dName
			mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

			// Create pool of packs.
			numLines := 4 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			// Expect InputRunner calls to get InChan and inject outgoing msgs
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
			// Expect calls to get decoder and decode each message. Since the
			// decoding is a no-op, the message payload will be the log file
			// line, unchanged.
			ith.MockInputRunner.EXPECT().Name().Return("LogfileInput")
			pbcall := ith.MockHelper.EXPECT().DecoderRunner(dName, "LogfileInput-"+dName)
			pbcall.Return(mockDecoderRunner, true)
			decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
			decodeCall.Return(ith.DecodeChan)

			go func() {
				err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
			}()
			for x := 0; x < numLines; x++ {
				_ = <-ith.DecodeChan
				// Free up the scheduler while we wait for the log file lines
				// to be processed.
				runtime.Gosched()
			}

			newFM := new(FileMonitor)
			newFM.Init(lfiConfig)
			c.Expect(err, gs.Equals, nil)

			fbytes, _ := json.Marshal(lfInput.Monitor)

			// Check that the persisted hashcode is from the last
			// complete log line
			expected_lastline := `10.1.1.4 plinko-565.byzantium.mozilla.com user3 [15/Mar/2013:12:20:27 -0700] "GET /1.1/user3/storage/passwords?newer=1356237662.44&full=1 HTTP/1.1" 200 1396 "-" "Firefox/20.0.1 FxSync/1.22.0.201304.desktop" "-" "ssl: SSL_RSA_WITH_RC4_128_SHA, version=TLSv1, bits=128" node_s:0.047167 req_s:0.047167 retries:0 req_b:446 "c_l:-"` + "\n"

			c.Expect((strings.IndexAny(string(fbytes),
				sha1_hexdigest(expected_lastline)) > -1), gs.IsTrue)

			json.Unmarshal(fbytes, &newFM)

			if runtime.GOOS == "windows" {
				c.Expect(newFM.seek, gs.Equals, int64(1253))
			} else {
				c.Expect(newFM.seek, gs.Equals, int64(1249))
			}

			lfInput.Stop()
		})
	})

	c.Specify("A LogfileDirectoryManagerInput", func() {
		c.Specify("empty file name", func() {
			var err error

			ldm := new(LogfileDirectoryManagerInput)
			conf := ldm.ConfigStruct().(*LogfileInputConfig)
			conf.LogFile = ""
			err = ldm.Init(conf)
			c.Expect(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), gs.Equals, "A logfile name must be specified.")
		})

		c.Specify("glob in file name", func() {
			var err error

			ldm := new(LogfileDirectoryManagerInput)
			conf := ldm.ConfigStruct().(*LogfileInputConfig)
			conf.LogFile = "../testsupport/*.log"
			err = ldm.Init(conf)
			c.Expect(err, gs.Not(gs.IsNil))
			c.Expect(err.Error(), gs.Equals, "Globs are not allowed in the file name: *.log")
		})

		// Note: Testing the actual functionality (spinning up new plugins within Heka)
		// is a manual process.
	})
}
Ejemplo n.º 11
0
func FileMonitorSpec(c gs.Context) {
	config := NewPipelineConfig(nil)
	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests-")
	c.Expect(tmpErr, gs.Equals, nil)
	origBaseDir := Globals().BaseDir
	Globals().BaseDir = tmpDir
	defer func() {
		Globals().BaseDir = origBaseDir
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.Equals, nil)
	}()
	journalName := "test-seekjournal"
	journalDir := filepath.Join(tmpDir, "seekjournals")
	tmpErr = os.MkdirAll(journalDir, 0770)
	c.Expect(tmpErr, gs.Equals, nil)
	journalPath := filepath.Join(journalDir, journalName)

	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("saved last read position", func() {

		c.Specify("without a previous journal", func() {
			ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
			lfInput, lfiConfig := createLogfileInput(journalName)

			// Initialize the input test helper
			err := lfInput.Init(lfiConfig)
			c.Expect(err, gs.IsNil)

			dName := "decoder-name"
			lfInput.decoderName = dName
			mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

			// Create pool of packs.
			numLines := 95 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			// Expect InputRunner calls to get InChan and inject outgoing msgs
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
			// Expect calls to get decoder and decode each message. Since the
			// decoding is a no-op, the message payload will be the log file
			// line, unchanged.
			ith.MockInputRunner.EXPECT().Name().Return("FileMonitor")
			ith.MockHelper.EXPECT().DecoderRunner(dName, "FileMonitor-"+dName).Return(mockDecoderRunner, true)
			decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
			decodeCall.Return(ith.DecodeChan)
			go func() {
				err = lfInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
			}()
			for x := 0; x < numLines; x++ {
				_ = <-ith.DecodeChan
				// Free up the scheduler while we wait for the log file lines
				// to be processed.
				runtime.Gosched()
			}

			newFM := new(FileMonitor)
			newFM.Init(lfiConfig)
			c.Expect(err, gs.Equals, nil)

			fbytes, _ := json.Marshal(lfInput.Monitor)
			json.Unmarshal(fbytes, &newFM)
			c.Expect(newFM.seek, gs.Equals, int64(28950))
			lfInput.Stop()
		})

		c.Specify("with a previous journal initializes with a seek value", func() {
			lfInput, lfiConfig := createLogfileInput(journalName)

			journalData := `{"last_hash":"f0b60af7f2cb35c3724151422e2f999af6e21fc0","last_start":28650,"last_len":300,"seek":28950}`

			journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660)
			c.Expect(journalErr, gs.Equals, nil)

			journal.WriteString(journalData)
			journal.Close()

			err := lfInput.Init(lfiConfig)
			c.Expect(err, gs.IsNil)

			c.Expect(err, gs.IsNil)

			// # bytes should be set to what's in the journal data
			c.Expect(lfInput.Monitor.seek, gs.Equals, int64(28950))
		})

		c.Specify("resets last read position to 0 if hash doesn't match", func() {
			lfInput, lfiConfig := createLogfileInput(journalName)
			lfiConfig.ResumeFromStart = true

			journalData := `{"last_hash":"xxxxx","last_start":28650,"last_len":300,"seek":28950}`

			journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660)
			c.Expect(journalErr, gs.Equals, nil)

			journal.WriteString(journalData)
			journal.Close()

			err := lfInput.Init(lfiConfig)
			c.Expect(err, gs.IsNil)

			// # bytes should be set to what's in the journal data
			c.Expect(lfInput.Monitor.seek, gs.Equals, int64(0))
		})

		c.Specify("resets last read position to end of file if hash doesn't match", func() {
			lfInput, lfiConfig := createLogfileInput(journalName)
			lfiConfig.ResumeFromStart = false

			journalData := `{"last_hash":"xxxxx","last_start":28650,"last_len":300,"seek":28950}`

			journal, journalErr := os.OpenFile(journalPath, os.O_CREATE|os.O_RDWR, 0660)
			c.Expect(journalErr, gs.Equals, nil)

			journal.WriteString(journalData)
			journal.Close()

			err := lfInput.Init(lfiConfig)

			c.Expect(err, gs.IsNil)

			// # bytes should be set to what's in the journal data
			c.Expect(lfInput.Monitor.seek, gs.Equals, int64(28950))
		})

	})

	c.Specify("filemonitor generates expected journal path", func() {
		lfInput := new(LogfileInput)
		lfiConfig := lfInput.ConfigStruct().(*LogfileInputConfig)
		lfiConfig.LogFile = filepath.Join("..", "testsupport", "test-zeus.log")
		lfiConfig.DiscoverInterval = 5
		lfiConfig.StatInterval = 5

		err := lfInput.Init(lfiConfig)
		c.Expect(err, gs.Equals, nil)
		clean := filepath.Join(journalDir, "___testsupport_test-zeus_log")
		c.Expect(lfInput.Monitor.seekJournalPath, gs.Equals, clean)
	})
}