Example #1
0
func (h *HekaFramingSplitter) UnframeRecord(framed []byte, pack *PipelinePack) []byte {
	headerLen := int(framed[1]) + message.HEADER_FRAMING_SIZE
	unframed := framed[headerLen:]
	if !h.SkipAuth && headerLen > message.UUID_SIZE {
		header := &message.Header{}
		decoded, err := message.DecodeHeader(framed[2:headerLen], header)
		if err != nil {
			h.sr.LogError(err)
		}
		if decoded && authenticateMessage(h.Signers, header, unframed) {
			pack.Signer = header.GetHmacSigner()
		} else {
			return nil
		}
	}
	return unframed
}
Example #2
0
func (h *HekaFramingSplitter) FindRecord(buf []byte) (bytesRead int, record []byte) {
	bytesRead = bytes.IndexByte(buf, message.RECORD_SEPARATOR)
	if bytesRead == -1 {
		bytesRead = len(buf)
		return // read more data to find the start of the next message
	}

	if len(buf) < bytesRead+message.HEADER_DELIMITER_SIZE {
		return // read more data to get the header length byte
	}
	headerLength := int(buf[bytesRead+1])
	headerEnd := bytesRead + headerLength + message.HEADER_FRAMING_SIZE
	if len(buf) < headerEnd {
		return // read more data to get the remainder of the header
	}
	decoded, err := message.DecodeHeader(
		buf[bytesRead+message.HEADER_DELIMITER_SIZE:headerEnd], h.header)
	if err != nil {
		h.sr.LogError(err)
	}
	if h.header.MessageLength != nil || decoded {
		messageEnd := headerEnd + int(h.header.GetMessageLength())
		if len(buf) < messageEnd {
			return // read more data to get the remainder of the message
		}
		record = buf[bytesRead:messageEnd]
		bytesRead = messageEnd
		h.header.Reset()
	} else {
		var n int
		bytesRead++                               // advance over the current record separator
		n, record = h.FindRecord(buf[bytesRead:]) // header was invalid, look again
		bytesRead += n
	}
	return bytesRead, record
}
Example #3
0
func OutputRunnerSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mockHelper := NewMockPluginHelper(ctrl)

	c.Specify("A runner", func() {
		stopoutputTimes = 0
		pConfig := NewPipelineConfig(nil)
		output := &StoppingOutput{}
		commonFO := CommonFOConfig{
			Matcher: "TRUE",
		}
		chanSize := 10
		maker := &pluginMaker{}
		maker.configStruct = make(map[string]interface{})
		pConfig.makers["Output"]["stoppingOutput"] = maker

		c.Specify("restarts a plugin on the first time only", func() {
			commonFO.Retries = RetryOptions{
				MaxDelay:   "1us",
				Delay:      "1us",
				MaxJitter:  "1us",
				MaxRetries: 1,
			}
			oRunner, err := NewFORunner("stoppingOutput", output, commonFO, "StoppingOutput",
				chanSize)
			c.Assume(err, gs.IsNil)
			oRunner.maker = maker

			close(oRunner.inChan) // signal the shutdown
			mockHelper.EXPECT().PipelineConfig().Return(pConfig)
			var wg sync.WaitGroup
			wg.Add(1)
			oRunner.Start(mockHelper, &wg) // no panic => success
			wg.Wait()
			c.Expect(stopoutputTimes, gs.Equals, 2)
		})

		c.Specify("restarts plugin and resumes feeding it", func() {
			output := &StopResumeOutput{}
			commonFO.Retries = RetryOptions{
				MaxDelay:   "1us",
				Delay:      "1us",
				MaxJitter:  "1us",
				MaxRetries: 4,
			}
			oRunner, err := NewFORunner("stoppingOutput", output, commonFO,
				"StoppingResumeOutput", chanSize)
			c.Assume(err, gs.IsNil)
			oRunner.maker = maker

			close(oRunner.inChan) // signal the shutdown
			mockHelper.EXPECT().PipelineConfig().Return(pConfig)
			var wg sync.WaitGroup
			wg.Add(1)
			oRunner.Start(mockHelper, &wg) // no panic => success
			wg.Wait()
			c.Expect(stopresumerunTimes, gs.Equals, 3)
			c.Expect(len(stopresumeHolder), gs.Equals, 2)
			c.Expect(stopresumeHolder[1], gs.Equals, "woot")
			c.Expect(oRunner.retainPack, gs.IsNil)
		})

		c.Specify("can exit without causing shutdown", func() {
			commonFO.Retries = RetryOptions{MaxRetries: 0}
			oRunner, err := NewFORunner("stoppingOutput", output, commonFO, "StoppingOutput",
				chanSize)
			c.Assume(err, gs.IsNil)
			oRunner.canExit = true
			oRunner.maker = maker

			// This pack is for the sending of the terminated message
			pack := NewPipelinePack(pConfig.injectRecycleChan)
			pConfig.injectRecycleChan <- pack

			// Feed in a pack to the input so we can verify its been recycled
			// after stopping (no leaks)
			pack = NewPipelinePack(pConfig.inputRecycleChan)
			oRunner.inChan <- pack

			// This is code to emulate the router removing the Output, and
			// closing up the outputs inChan channel
			go func() {
				<-pConfig.Router().RemoveOutputMatcher()
				// We don't close the matcher inChan because its not
				// instantiated in the tests
				close(oRunner.inChan)
			}()

			mockHelper.EXPECT().PipelineConfig().Return(pConfig)
			var wg sync.WaitGroup
			wg.Add(1)
			oRunner.Start(mockHelper, &wg)
			wg.Wait()
			c.Expect(stopoutputTimes, gs.Equals, 1)
			p := <-pConfig.router.inChan
			c.Expect(p.Message.GetType(), gs.Equals, "heka.terminated")
			c.Expect(p.Message.GetLogger(), gs.Equals, "hekad")
			plugin, _ := p.Message.GetFieldValue("plugin")
			c.Expect(plugin, gs.Equals, "stoppingOutput")
			// This should be 1 because the inChan should be flushed
			// and packs should not be leaked
			c.Expect(len(pConfig.inputRecycleChan), gs.Equals, 1)
		})

		c.Specify("encodes a message", func() {
			oRunner, err := NewFORunner("stoppingOutput", output, commonFO, "StoppingOutput",
				chanSize)
			c.Assume(err, gs.IsNil)
			oRunner.encoder = new(_payloadEncoder)
			oRunner.maker = maker
			_pack.Message = ts.GetTestMessage()
			payload := "Test Payload"

			c.Specify("without framing", func() {
				result, err := oRunner.Encode(_pack)
				c.Expect(err, gs.IsNil)
				c.Expect(string(result), gs.Equals, payload)
			})

			c.Specify("with framing", func() {
				oRunner.SetUseFraming(true)
				result, err := oRunner.Encode(_pack)
				c.Expect(err, gs.IsNil)

				i := bytes.IndexByte(result, message.UNIT_SEPARATOR)
				c.Expect(i > 3, gs.IsTrue, -1)
				c.Expect(string(result[i+1:]), gs.Equals, payload)
				header := new(message.Header)
				ok, err := message.DecodeHeader(result[2:i+1], header)
				c.Expect(ok, gs.IsTrue)
				c.Expect(err, gs.IsNil)
				c.Expect(header.GetMessageLength(), gs.Equals, uint32(len(payload)))
			})

			c.Specify("with framing, ignore message", func() {
				oRunner.SetUseFraming(true)
				oRunner.encoder = new(_ignoreEncoder)
				result, err := oRunner.Encode(_pack)
				c.Expect(err, gs.IsNil)
				c.Expect(result == nil, gs.IsTrue)
			})
		})
	})
}