Exemple #1
0
func BenchmarkMultiDecodeProtobuf(b *testing.B) {
	b.StopTimer()
	pConfig := NewPipelineConfig(nil) // initializes Globals
	msg := pipeline_ts.GetTestMessage()
	msg.SetPayload("This is a test")
	pack := NewPipelinePack(pConfig.InputRecycleChan())
	pack.MsgBytes, _ = proto.Marshal(msg)
	decoder := new(MultiDecoder)
	decoder.SetPipelineConfig(pConfig)
	conf := decoder.ConfigStruct().(*MultiDecoderConfig)
	sub := new(ProtobufDecoder)
	sub.SetPipelineConfig(pConfig)
	sub.Init(nil)
	wrapper0 := NewPluginWrapper("sub", pConfig)
	wrapper0.CreateWithError = func() (interface{}, error) {
		return sub, nil
	}
	pConfig.DecoderWrappers["sub"] = wrapper0
	conf.CascadeStrategy = "first-wins"
	conf.Subs = []string{"sub"}
	decoder.Init(conf)

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		decoder.Decode(pack)
	}
}
Exemple #2
0
func ProtobufDecoderSpec(c gospec.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	msg := ts.GetTestMessage()
	config := NewPipelineConfig(nil) // Initializes globals.

	c.Specify("A ProtobufDecoder", func() {
		encoded, err := proto.Marshal(msg)
		c.Assume(err, gs.IsNil)
		pack := NewPipelinePack(config.inputRecycleChan)
		decoder := new(ProtobufDecoder)
		decoder.sampleDenominator = 1000 // Since we don't call decoder.Init().

		c.Specify("decodes a protobuf message", func() {
			pack.MsgBytes = encoded
			_, err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(pack.Message, gs.Equals, msg)
			v, ok := pack.Message.GetFieldValue("foo")
			c.Expect(ok, gs.IsTrue)
			c.Expect(v, gs.Equals, "bar")
		})

		c.Specify("returns an error for bunk encoding", func() {
			bunk := append([]byte{0, 0, 0}, encoded...)
			pack.MsgBytes = bunk
			_, err := decoder.Decode(pack)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
Exemple #3
0
func BenchmarkMultiDecodeProtobuf(b *testing.B) {
	b.StopTimer()
	pConfig := NewPipelineConfig(nil) // initializes Globals
	msg := pipeline_ts.GetTestMessage()
	msg.SetPayload("This is a test")
	pack := NewPipelinePack(pConfig.InputRecycleChan())
	pack.MsgBytes, _ = proto.Marshal(msg)
	decoder := new(MultiDecoder)
	decoder.SetPipelineConfig(pConfig)
	conf := decoder.ConfigStruct().(*MultiDecoderConfig)

	RegisterPlugin("ProtobufDecoder", func() interface{} {
		return &ProtobufDecoder{}
	})
	defer delete(AvailablePlugins, "ProtobufDecoder")
	var section PluginConfig
	_, err := toml.Decode("", &section)
	if err != nil {
		b.Fatalf("Error decoding empty TOML: %s", err.Error())
	}
	maker, err := NewPluginMaker("ProtobufDecoder", pConfig, section)
	if err != nil {
		b.Fatalf("Error decoding empty TOML: %s", err.Error())
	}
	pConfig.DecoderMakers["ProtobufDecoder"] = maker

	conf.CascadeStrategy = "first-wins"
	conf.Subs = []string{"sub"}
	decoder.Init(conf)

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		decoder.Decode(pack)
	}
}
Exemple #4
0
func FilterRunnerSpec(c gs.Context) {
	c.Specify("A filterrunner", func() {
		pConfig := NewPipelineConfig(nil)
		filter := &CounterFilter{}
		commonFO := CommonFOConfig{
			Matcher: "Type == 'bogus'",
		}
		chanSize := 10
		fRunner, err := NewFORunner("counterFilter", filter, commonFO, "CounterFilter",
			chanSize)
		fRunner.h = pConfig
		c.Assume(err, gs.IsNil)

		pack := NewPipelinePack(pConfig.injectRecycleChan)
		pConfig.injectRecycleChan <- pack
		pack.Message = ts.GetTestMessage()
		c.Assume(pack.TrustMsgBytes, gs.IsFalse)
		msgEncoding, err := proto.Marshal(pack.Message)
		c.Assume(err, gs.IsNil)

		c.Specify("puts protobuf encoding into MsgBytes before delivery", func() {
			result := fRunner.Inject(pack)
			c.Expect(result, gs.IsTrue)
			recd := <-pConfig.router.inChan
			c.Expect(recd, gs.Equals, pack)
			c.Expect(recd.TrustMsgBytes, gs.IsTrue)
			c.Expect(bytes.Equal(msgEncoding, recd.MsgBytes), gs.IsTrue)
		})
	})
}
func StatsdInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
	ith.PackSupply = make(chan *PipelinePack, 1)

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, input runner, and stat accumulator
	ith.MockHelper = NewMockPluginHelper(ctrl)
	ith.MockInputRunner = NewMockInputRunner(ctrl)
	mockStatAccum := NewMockStatAccumulator(ctrl)

	c.Specify("A StatsdInput", func() {
		statsdInput := StatsdInput{}
		config := statsdInput.ConfigStruct().(*StatsdInputConfig)

		config.Address = ith.AddrStr
		err := statsdInput.Init(config)
		c.Assume(err, gs.IsNil)
		realListener := statsdInput.listener
		c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()
		mockListener := pipeline_ts.NewMockConn(ctrl)
		statsdInput.listener = mockListener

		ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil)
		mockListener.EXPECT().Close()
		mockListener.EXPECT().SetReadDeadline(gomock.Any())

		c.Specify("sends a Stat to the StatAccumulator", func() {
			statName := "sample.count"
			statVal := 303
			msg := fmt.Sprintf("%s:%d|c\n", statName, statVal)
			expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)}
			mockStatAccum.EXPECT().DropStat(expected).Return(true)
			readCall := mockListener.EXPECT().Read(make([]byte, 512))
			readCall.Return(len(msg), nil)
			readCall.Do(func(msgBytes []byte) {
				copy(msgBytes, []byte(msg))
				statsdInput.Stop()
			})
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()
			wg.Wait()
		})
	})
}
Exemple #6
0
func BenchmarkEncodeProtobuf(b *testing.B) {
	b.StopTimer()
	msg := ts.GetTestMessage()
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		proto.Marshal(msg)
	}
}
func MessageTemplateSpec(c gs.Context) {
	c.Specify("A message template", func() {
		mt := make(MessageTemplate)
		mt["Logger"] = "test logger"
		mt["Payload"] = "test payload"
		mt["Hostname"] = "host.example.com"
		mt["Pid"] = "123.456"
		mt["Type"] = "test type"
		mt["Severity"] = "23"
		mt["tmplTest|baz"] = "bar"
		msg := ts.GetTestMessage()

		c.Specify("replaces message values", func() {
			err := mt.PopulateMessage(msg, nil)
			c.Assume(err, gs.IsNil)
			c.Expect(msg.GetLogger(), gs.Equals, mt["Logger"])
			c.Expect(msg.GetPayload(), gs.Equals, mt["Payload"])
			c.Expect(msg.GetHostname(), gs.Equals, mt["Hostname"])
			c.Expect(msg.GetPid(), gs.Equals, int32(123))
			c.Expect(msg.GetType(), gs.Equals, mt["Type"])
			c.Expect(msg.GetSeverity(), gs.Equals, int32(23))
			fields := msg.FindAllFields("tmplTest")
			c.Expect(len(fields), gs.Equals, 1)
			field := fields[0]
			value := field.GetValueString()
			c.Expect(len(value), gs.Equals, 1)
			c.Expect(value[0], gs.Equals, "bar")
			c.Expect(field.GetRepresentation(), gs.Equals, "baz")
		})

		c.Specify("honors substitutions", func() {
			mt["Payload"] = "this is %substitution%"
			mt["Hostname"] = "%host%.example.com"
			mt["Type"] = "another %substitution%"
			mt["tmplTest|baz"] = "%fieldvalue%"
			subs := make(map[string]string)
			subs["host"] = "otherhost"
			subs["substitution"] = "a test"
			subs["fieldvalue"] = "wakajawaka"
			err := mt.PopulateMessage(msg, subs)
			c.Assume(err, gs.IsNil)
			c.Expect(msg.GetLogger(), gs.Equals, mt["Logger"])
			c.Expect(msg.GetPayload(), gs.Equals, "this is a test")
			c.Expect(msg.GetHostname(), gs.Equals, "otherhost.example.com")
			c.Expect(msg.GetPid(), gs.Equals, int32(123))
			c.Expect(msg.GetType(), gs.Equals, "another a test")
			c.Expect(msg.GetSeverity(), gs.Equals, int32(23))
			fields := msg.FindAllFields("tmplTest")
			c.Expect(len(fields), gs.Equals, 1)
			field := fields[0]
			value := field.GetValueString()
			c.Expect(len(value), gs.Equals, 1)
			c.Expect(value[0], gs.Equals, subs["fieldvalue"])
			c.Expect(field.GetRepresentation(), gs.Equals, "baz")
		})
	})
}
Exemple #8
0
func BenchmarkDecodeProtobuf(b *testing.B) {
	b.StopTimer()
	msg := ts.GetTestMessage()
	encoded, _ := proto.Marshal(msg)
	config := NewPipelineConfig(nil)
	pack := NewPipelinePack(config.inputRecycleChan)
	decoder := new(ProtobufDecoder)
	pack.MsgBytes = encoded
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		decoder.Decode(pack)
	}
}
Exemple #9
0
func TcpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)
	key := "testkey"
	signers := map[string]Signer{"test_1": {key}}
	signer := "test"

	c.Specify("A TcpInput protobuf parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
		tcpInput := TcpInput{}
		err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
			Signers:    signers,
			Decoder:    "ProtobufDecoder",
			ParserType: "message.proto"})
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr)
		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		enccall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "TcpInput-123-ProtobufDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		cleanup := func() {
			mockListener.EXPECT().Close()
			tcpInput.Stop()
			tcpInput.wg.Wait()
		}

		c.Specify("reads a message from its connection", func() {
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
		})

		c.Specify("reads a MD5 signed message from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(md5.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-ith.DecodeChan:
				c.Expect(ith.Pack, gs.Equals, packRef)
				c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
				c.Expect(ith.Pack.Signer, gs.Equals, "test")
			case t := <-timeout:
				c.Expect(t, gs.IsNil)
			}
		})

		c.Specify("reads a SHA1 signed message from its connection", func() {
			header.SetHmacHashFunction(message.Header_SHA1)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(sha1.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-ith.DecodeChan:
				c.Expect(ith.Pack, gs.Equals, packRef)
				c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
				c.Expect(ith.Pack.Signer, gs.Equals, "test")
			case t := <-timeout:
				c.Expect(t, gs.IsNil)
			}
		})

		c.Specify("reads a signed message with an expired key from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(11)) // non-existent key version
			hm := hmac.New(md5.New, []byte(key))
			hm.Write(mbytes)
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-mockDecoderRunner.InChan():
				c.Expect(packRef, gs.IsNil)
			case t := <-timeout:
				c.Expect(t, gs.IsTrue)
			}
		})

		c.Specify("reads a signed message with an incorrect hmac from its connection", func() {
			header.SetHmacHashFunction(message.Header_MD5)
			header.SetHmacSigner(signer)
			header.SetHmacKeyVersion(uint32(1))
			hm := hmac.New(md5.New, []byte(key))
			hm.Write([]byte("some bytes"))
			header.SetHmac(hm.Sum(nil))
			hbytes, _ := proto.Marshal(header)
			buflen := 3 + len(hbytes) + len(mbytes)
			readCall.Return(buflen, nil)
			readCall.Do(getPayloadBytes(hbytes, mbytes))

			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer cleanup()
			ith.PackSupply <- ith.Pack
			timeout := make(chan bool, 1)
			go func() {
				time.Sleep(100 * time.Millisecond)
				timeout <- true
			}()
			select {
			case packRef := <-mockDecoderRunner.InChan():
				c.Expect(packRef, gs.IsNil)
			case t := <-timeout:
				c.Expect(t, gs.IsTrue)
			}
		})
	})

	c.Specify("A TcpInput regexp parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")

		config := &TcpInputConfig{
			Net:        "tcp",
			Address:    ith.AddrStr,
			Decoder:    "RegexpDecoder",
			ParserType: "regexp",
		}

		tcpInput := TcpInput{}
		err := tcpInput.Init(config)
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
		mbytes := []byte("this is a test message\n")
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Name().Return("logger")
		enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "TcpInput-123-RegexpDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		c.Specify("reads a message from its connection", func() {
			readCall.Return(len(mbytes), nil)
			readCall.Do(getPayloadText(mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				mockListener.EXPECT().Close()
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes[:len(mbytes)-1]))
			c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
			c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
		})
	})

	c.Specify("A TcpInput token parser", func() {
		ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
		tcpInput := TcpInput{}
		err := tcpInput.Init(&TcpInputConfig{Net: "tcp", Address: ith.AddrStr,
			Decoder:    "TokenDecoder",
			ParserType: "token",
			Delimiter:  "\n"})
		c.Assume(err, gs.IsNil)
		realListener := tcpInput.listener
		c.Expect(realListener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()

		mockConnection := pipeline_ts.NewMockConn(ctrl)
		mockListener := pipeline_ts.NewMockListener(ctrl)
		tcpInput.listener = mockListener

		addr := new(address)
		addr.str = "123"
		mockConnection.EXPECT().RemoteAddr().Return(addr).Times(2)
		mbytes := []byte("this is a test message\n")
		err = errors.New("connection closed") // used in the read return(s)
		readCall := mockConnection.EXPECT().Read(gomock.Any())
		readEnd := mockConnection.EXPECT().Read(gomock.Any()).After(readCall)
		readEnd.Return(0, err)
		mockConnection.EXPECT().SetReadDeadline(gomock.Any()).Return(nil).AnyTimes()
		mockConnection.EXPECT().Close()

		neterr := pipeline_ts.NewMockError(ctrl)
		neterr.EXPECT().Temporary().Return(false)
		acceptCall := mockListener.EXPECT().Accept().Return(mockConnection, nil)
		acceptCall.Do(func() {
			acceptCall = mockListener.EXPECT().Accept()
			acceptCall.Return(nil, neterr)
		})

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Name().Return("logger")
		enccall := ith.MockHelper.EXPECT().DecoderRunner("TokenDecoder", "TcpInput-123-TokenDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)
		ith.MockHelper.EXPECT().StopDecoderRunner(ith.Decoder)

		c.Specify("reads a message from its connection", func() {
			readCall.Return(len(mbytes), nil)
			readCall.Do(getPayloadText(mbytes))
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				mockListener.EXPECT().Close()
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, string(mbytes))
			c.Expect(ith.Pack.Message.GetLogger(), gs.Equals, "logger")
			c.Expect(ith.Pack.Message.GetHostname(), gs.Equals, "123")
		})
	})

	c.Specify("A TcpInput using TLS", func() {
		tcpInput := TcpInput{}
		config := &TcpInputConfig{
			Net:        "tcp",
			Address:    ith.AddrStr,
			ParserType: "token",
			UseTls:     true,
		}

		c.Specify("fails to init w/ missing key or cert file", func() {
			config.Tls = TlsConfig{}
			err := tcpInput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("accepts TLS client connections", func() {
			ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
			config.Tls = TlsConfig{
				CertFile: "./testsupport/cert.pem",
				KeyFile:  "./testsupport/key.pem",
			}
			err := tcpInput.Init(config)
			c.Expect(err, gs.IsNil)
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				tcpInput.Stop()
				tcpInput.wg.Wait()
			}()

			clientConfig := new(tls.Config)
			clientConfig.InsecureSkipVerify = true
			conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
			c.Expect(err, gs.IsNil)
			defer conn.Close()
			conn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
			n, err := conn.Write([]byte("This is a test."))
			c.Expect(err, gs.IsNil)
			c.Expect(n, gs.Equals, len("This is a test."))
		})

		c.Specify("doesn't accept connections below specified min TLS version", func() {
			ith.MockInputRunner.EXPECT().Name().Return("TcpInput")
			config.Tls = TlsConfig{
				CertFile:   "./testsupport/cert.pem",
				KeyFile:    "./testsupport/key.pem",
				MinVersion: "TLS12",
			}
			err := tcpInput.Init(config)
			c.Expect(err, gs.IsNil)
			go tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			defer func() {
				tcpInput.Stop()
				tcpInput.wg.Wait()
				time.Sleep(time.Duration(1000))
			}()

			clientConfig := &tls.Config{
				InsecureSkipVerify: true,
				MaxVersion:         tls.VersionTLS11,
			}
			conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
			c.Expect(conn, gs.IsNil)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
func DashboardOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := pipeline.NewPipelineConfig(nil)

	dashboardOutput := new(DashboardOutput)
	dashboardOutput.pConfig = pConfig

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)

	errChan := make(chan error, 1)

	startOutput := func() {
		go func() {
			errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
		}()
	}

	if runtime.GOOS != "windows" {
		c.Specify("A DashboardOutput", func() {

			tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
			c.Assume(err, gs.IsNil)
			config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
			config.WorkingDirectory = tmpdir

			c.Specify("Init halts if basedirectory is not writable", func() {
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)
				err = dashboardOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
			})

			c.Specify("that is running", func() {
				startedChan := make(chan bool, 1)
				defer close(startedChan)
				ts := httptest.NewUnstartedServer(nil)

				dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
					ts.Start()
					startedChan <- true
					return nil
				}

				ticker := make(chan time.Time)
				inChan := make(chan *pipeline.PipelinePack, 1)
				recycleChan := make(chan *pipeline.PipelinePack, 1)
				pack := pipeline.NewPipelinePack(recycleChan)
				pack.Message = pipeline_ts.GetTestMessage()

				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)

				err := os.MkdirAll(tmpdir, 0700)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)

				dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					// Noop
				})

				c.Specify("sets custom http headers", func() {
					config.Headers = http.Header{
						"One":  []string{"two", "three"},
						"Four": []string{"five", "six", "seven"},
					}
					err = dashboardOutput.Init(config)
					c.Assume(err, gs.IsNil)
					ts.Config = dashboardOutput.server

					startOutput()

					inChan <- pack
					<-startedChan
					resp, err := http.Get(ts.URL)
					c.Assume(err, gs.IsNil)
					resp.Body.Close()
					c.Assume(resp.StatusCode, gs.Equals, 200)

					// Verify headers are there
					eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
					c.Expect(eq, gs.IsTrue)
					eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
					c.Expect(eq, gs.IsTrue)
				})

				close(inChan)
				c.Expect(<-errChan, gs.IsNil)

				ts.Close()
			})
		})
	}
}
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	pConfig := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network.
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// Set up mock helper, runner, and pack supply channel.
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: pConfig}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs.
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
				ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
					ith.MockSplitterRunner)
				ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
				ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
				ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

				getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
					gomock.Any()).Times(numLines)
				line := "boo hoo foo foo"
				getRecCall.Return(len(line), []byte(line), nil)
				getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
				getRecCall.Return(0, make([]byte, 0), io.EOF)

				deliverChan := make(chan []byte, 1)
				deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
					ith.MockDeliverer).Times(numLines)
				deliverCall.Do(func(record []byte, del Deliverer) {
					deliverChan <- record
				})

				ith.MockDeliverer.EXPECT().Done()

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				dur, _ := time.ParseDuration("5s")
				timeout := time.After(dur)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case record := <-deliverChan:
						c.Expect(string(record), gs.Equals, line)
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Exemple #12
0
func ProcessInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A ProcessInput", func() {
		pInput := ProcessInput{}

		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).AnyTimes()
		ith.MockInputRunner.EXPECT().Name().Return("logger").AnyTimes()

		enccall := ith.MockHelper.EXPECT().DecoderRunner("RegexpDecoder", "logger-RegexpDecoder").AnyTimes()
		enccall.Return(ith.Decoder, true)

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).AnyTimes()

		config := pInput.ConfigStruct().(*ProcessInputConfig)
		config.Command = make(map[string]cmdConfig)

		pConfig := NewPipelineConfig(nil)
		ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

		tickChan := make(chan time.Time)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

		c.Specify("reads a message from ProcessInput", func() {

			pInput.SetName("SimpleTest")
			config.Decoder = "RegexpDecoder"
			config.ParserType = "token"
			config.Delimiter = "|"

			// Note that no working directory is explicitly specified
			config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_TEST1_CMD, Args: PROCESSINPUT_TEST1_CMD_ARGS}
			err := pInput.Init(config)
			c.Assume(err, gs.IsNil)

			go func() {
				pInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			tickChan <- time.Now()

			expected_payloads := PROCESSINPUT_TEST1_OUTPUT
			actual_payloads := []string{}

			for x := 0; x < 4; x++ {
				ith.PackSupply <- ith.Pack
				packRef := <-ith.DecodeChan
				c.Expect(ith.Pack, gs.Equals, packRef)
				actual_payloads = append(actual_payloads, *packRef.Message.Payload)
				fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
				c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")
				// Free up the scheduler
				runtime.Gosched()
			}

			for x := 0; x < 4; x++ {
				c.Expect(expected_payloads[x], gs.Equals, actual_payloads[x])
			}

			pInput.Stop()
		})

		c.Specify("handles bad arguments", func() {

			pInput.SetName("BadArgs")
			config.ParseStdout = false
			config.ParseStderr = true
			config.Decoder = "RegexpDecoder"
			config.ParserType = "token"
			config.Delimiter = "|"

			// Note that no working directory is explicitly specified
			config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS}

			err := pInput.Init(config)
			c.Assume(err, gs.IsNil)

			expected_err := fmt.Errorf("BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]")
			ith.MockInputRunner.EXPECT().LogError(expected_err)

			go func() {
				pInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			tickChan <- time.Now()

			ith.PackSupply <- ith.Pack
			<-ith.DecodeChan
			runtime.Gosched()

			pInput.Stop()
		})

		c.Specify("can pipe multiple commands together", func() {

			pInput.SetName("PipedCmd")
			config.Decoder = "RegexpDecoder"
			config.ParserType = "token"
			// Overload the delimiter
			config.Delimiter = " "

			// Note that no working directory is explicitly specified
			config.Command["0"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD1, Args: PROCESSINPUT_PIPE_CMD1_ARGS}
			config.Command["1"] = cmdConfig{Bin: PROCESSINPUT_PIPE_CMD2, Args: PROCESSINPUT_PIPE_CMD2_ARGS}
			err := pInput.Init(config)
			c.Assume(err, gs.IsNil)

			go func() {
				pInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			tickChan <- time.Now()

			expected_payloads := PROCESSINPUT_PIPE_OUTPUT
			actual_payloads := []string{}

			for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ {
				ith.PackSupply <- ith.Pack
				packRef := <-ith.DecodeChan
				c.Expect(ith.Pack, gs.Equals, packRef)
				actual_payloads = append(actual_payloads, *packRef.Message.Payload)
				fPInputName := *packRef.Message.FindFirstField("ProcessInputName")
				c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")
				// Free up the scheduler
				runtime.Gosched()
			}

			for x := 0; x < len(PROCESSINPUT_PIPE_OUTPUT); x++ {
				c.Expect(fmt.Sprintf("[%d] [%s] [%x]",
					len(actual_payloads[x]),
					actual_payloads[x],
					actual_payloads[x]),
					gs.Equals,
					fmt.Sprintf("[%d] [%s] [%x]",
						len(expected_payloads[x]),
						expected_payloads[x],
						expected_payloads[x]))
			}

			pInput.Stop()
		})

	})
}
Exemple #13
0
func SmtpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	encoder := new(plugins.PayloadEncoder)
	econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
	econfig.AppendNewlines = false
	encoder.Init(econfig)

	c.Specify("A SmtpOutput", func() {
		smtpOutput := new(SmtpOutput)

		config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
		config.SendTo = []string{"root"}

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true
		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)
		runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
		runnerName.Return("SmtpOutput")
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()

		c.Specify("send email payload message", func() {
			err := smtpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			smtpOutput.sendFunction = testSendMail

			outStr := "Write me out to the network"
			pack.Message.SetPayload(outStr)
			go func() {
				wg.Add(1)
				smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
			inChan <- pack
			close(inChan)
			wg.Wait()
		})
	})

	// Use this test with a real server
	//  c.Specify("Real SmtpOutput output", func() {
	//  	smtpOutput := new(SmtpOutput)
	//
	//  	config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
	//  	config.SendTo = []string{"root"}
	//
	//  	msg := pipeline_ts.GetTestMessage()
	//  	pack := NewPipelinePack(pConfig.InputRecycleChan())
	//  	pack.Message = msg
	//  	pack.Decoded = true
	//  	inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
	//  	inChanCall.Return(inChan)
	//  	runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
	//  	runnerName.Return("SmtpOutput")
	//      oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()
	//
	//  	c.Specify("send a real email essage", func() {
	//
	//  		err := smtpOutput.Init(config)
	//  		c.Assume(err, gs.IsNil)
	//
	//  		outStr := "Write me out to the network"
	//  		pack.Message.SetPayload(outStr)
	//  		go func() {
	//  			wg.Add(1)
	//  			smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
	//  			wg.Done()
	//  		}()
	//  		inChan <- pack
	//  		time.Sleep(1000) // allow time for the message output
	//  		close(inChan)
	//  		wg.Wait()
	//  		// manually check the mail
	//  	})
	//  })
}
Exemple #14
0
func UdpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	udpOutput := new(UdpOutput)
	config := udpOutput.ConfigStruct().(*UdpOutputConfig)

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	encoder := new(plugins.PayloadEncoder)
	encoder.Init(new(plugins.PayloadEncoderConfig))

	inChan := make(chan *pipeline.PipelinePack, 1)
	rChan := make(chan *pipeline.PipelinePack, 1)
	var wg sync.WaitGroup
	var rAddr net.Addr

	c.Specify("A UdpOutput", func() {
		msg := pipeline_ts.GetTestMessage()
		payload := "Write me out to the network."
		msg.SetPayload(payload)
		pack := pipeline.NewPipelinePack(rChan)
		pack.Message = msg

		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
		oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
		oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))

		c.Specify("using UDP", func() {
			addr := "127.0.0.1:45678"
			config.Address = addr
			ch := make(chan string, 1)

			collectData := func() {
				conn, err := net.ListenPacket("udp", addr)
				if err != nil {
					ch <- err.Error()
					return
				}

				ch <- "ready"
				b := make([]byte, 1000)
				var n int
				n, rAddr, _ = conn.ReadFrom(b)
				ch <- string(b[:n])
				conn.Close()
			}

			go collectData()
			result := <-ch // Wait for server to be ready.
			c.Assume(result, gs.Equals, "ready")

			c.Specify("writes out to the network", func() {
				err := udpOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				close(inChan)
				wg.Wait()
			})

			c.Specify("uses the specified local address", func() {
				config.LocalAddress = "localhost:12345"
				err := udpOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				c.Expect(rAddr.Network(), gs.Equals, "udp")
				c.Expect(rAddr.String(), gs.Equals, "127.0.0.1:12345")
				close(inChan)
				wg.Wait()
			})
		})

		c.Specify("using Unix datagrams", func() {
			if runtime.GOOS == "windows" {
				return
			}

			testUnixAddr := func() string {
				f, err := ioutil.TempFile("", "_heka_test_sock")
				c.Assume(err, gs.IsNil)
				addr := f.Name()
				f.Close()
				os.Remove(addr)
				return addr
			}

			config.Address = testUnixAddr()
			config.Net = "unixgram"
			ch := make(chan string, 1)
			var wg sync.WaitGroup
			var rAddr net.Addr

			collectData := func() {
				conn, err := net.ListenPacket("unixgram", config.Address)
				if err != nil {
					ch <- err.Error()
					return
				}

				ch <- "ready"
				b := make([]byte, 1000)
				var n int
				n, rAddr, _ = conn.ReadFrom(b)
				ch <- string(b[:n])
				conn.Close()
				err = os.Remove(config.Address)
				var errMsg string
				if err != nil {
					errMsg = err.Error()
				}
				ch <- errMsg
			}

			go collectData()
			result := <-ch // Wait for server to be ready.
			c.Assume(result, gs.Equals, "ready")

			c.Specify("writes out to the network", func() {
				err := udpOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				close(inChan)
				wg.Wait()
				result = <-ch
				c.Expect(result, gs.Equals, "")
			})

			c.Specify("uses the specified local address", func() {
				config.LocalAddress = testUnixAddr()
				err := udpOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				c.Expect(rAddr.Network(), gs.Equals, "unixgram")
				c.Expect(rAddr.String(), gs.Equals, config.LocalAddress)
				close(inChan)
				wg.Wait()
			})

		})
	})
	c.Specify("drop message contents if their size is bigger than allowed UDP datagram size", func() {
		huge_msg := pipeline_ts.GetTestMessage()
		payload := strings.Repeat("2", 131014)

		huge_msg.SetPayload(payload)

		huge_pack := pipeline.NewPipelinePack(rChan)
		huge_pack.Message = huge_msg

		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
		oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
		oth.MockOutputRunner.EXPECT().Encode(huge_pack).Return(encoder.Encode(huge_pack))
		oth.MockOutputRunner.EXPECT().LogError(fmt.Errorf("Message has exceeded allowed UDP data size: 131014 > 65507"))

		config.Address = "localhost:12345"
		err := udpOutput.Init(config)

		c.Assume(err, gs.IsNil)
		wg.Add(1)
		go func() {
			err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.IsNil)
			wg.Done()
		}()
		inChan <- huge_pack

		close(inChan)
		wg.Wait()
	})

	c.Specify("checks validation of of Maximum message size limit", func() {

		config.Address = "localhost:12345"
		config.MaxMessageSize = 100

		err := udpOutput.Init(config)

		c.Assume(err.Error(), gs.Equals, "Maximum message size can't be smaller than 512 bytes.")
	})
}
Exemple #15
0
func CarbonOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := NewCarbonTestHelper(ctrl)
	var wg sync.WaitGroup
	pConfig := NewPipelineConfig(nil)

	// make test data
	const count = 5
	lines := make([]string, count)
	baseTime := time.Now().UTC().Add(-10 * time.Second)
	for i := 0; i < count; i++ {
		statName := fmt.Sprintf("stats.name.%d", i)
		statTime := baseTime.Add(time.Duration(i) * time.Second)
		lines[i] = fmt.Sprintf("%s %d %d", statName, i*2, statTime.Unix())
	}
	submit_data := strings.Join(lines, "\n")
	expected_data := submit_data + "\n"

	// a helper to make new test packs
	newpack := func() *PipelinePack {
		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true
		pack.Message.SetPayload(submit_data)
		return pack
	}

	// collectDataTCP and collectDataUDP functions; when ready reports its port on
	// chPort, or error on chErr; when data is received it is reported on chData
	collectDataTCP := func(chPort chan<- int, chData chan<- string, chError chan<- error) {
		tcpaddr, err := net.ResolveTCPAddr("tcp", ":0")
		if err != nil {
			chError <- err
			return
		}

		listener, err := net.ListenTCP("tcp", tcpaddr)
		if err != nil {
			chError <- err
			return
		}
		chPort <- listener.Addr().(*net.TCPAddr).Port

		conn, err := listener.Accept()
		if err != nil {
			chError <- err
			return
		}

		b := make([]byte, 10000)
		n, err := conn.Read(b)
		if err != nil {
			chError <- err
			return
		}

		chData <- string(b[0:n])
	}

	collectDataUDP := func(chPort chan<- int, chData chan<- string, chError chan<- error) {
		udpaddr, err := net.ResolveUDPAddr("udp", ":0")
		if err != nil {
			chError <- err
			return
		}

		conn, err := net.ListenUDP("udp", udpaddr)
		if err != nil {
			chError <- err
			return
		}
		chPort <- conn.LocalAddr().(*net.UDPAddr).Port

		b := make([]byte, 10000)
		n, _, err := conn.ReadFromUDP(b)
		if err != nil {
			chError <- err
			return
		}
		chData <- string(b[0:n])
	}

	doit := func(protocol string, collectData collectFunc) {
		c.Specify("A CarbonOutput ", func() {
			inChan := make(chan *PipelinePack, 1)
			carbonOutput := new(CarbonOutput)
			config := carbonOutput.ConfigStruct().(*CarbonOutputConfig)
			pack := newpack()

			c.Specify("writes "+protocol+" to the network", func() {
				inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
				inChanCall.Return(inChan)

				chError := make(chan error, count)
				chPort := make(chan int, count)
				chData := make(chan string, count)
				go collectData(chPort, chData, chError)

			WAIT_FOR_DATA:
				for {
					select {
					case port := <-chPort:
						// data collection server is ready, start CarbonOutput
						config.Address = fmt.Sprintf("127.0.0.1:%d", port)
						config.Protocol = protocol
						err := carbonOutput.Init(config)
						c.Assume(err, gs.IsNil)
						go func() {
							wg.Add(1)
							carbonOutput.Run(oth.MockOutputRunner, oth.MockHelper)
							wg.Done()
						}()

						// Send the pack.
						inChan <- pack

					case data := <-chData:
						close(inChan)
						wg.Wait() // wait for close to finish, prevents intermittent test failures
						c.Expect(data, gs.Equals, expected_data)
						break WAIT_FOR_DATA

					case err := <-chError:
						// fail
						c.Assume(err, gs.IsNil)
						return
					}
				}
			})
		})
	}

	doit("tcp", collectDataTCP)
	doit("udp", collectDataUDP)
}
Exemple #16
0
func HttpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	var (
		reqMethod string
		reqBody   string
		reqHeader http.Header
		handleWg  sync.WaitGroup
		runWg     sync.WaitGroup
		delay     bool
	)

	handler := new(_test_handler)
	handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) {
		defer handleWg.Done()
		if delay {
			time.Sleep(time.Duration(2) * time.Millisecond)
		}
		// Capture request body for test assertions.
		p := make([]byte, req.ContentLength)
		_, err := req.Body.Read(p)
		c.Expect(err.Error(), gs.Equals, "EOF")
		reqBody = string(p)
		// Capture request method and headers for test assertions.
		reqMethod = req.Method
		reqHeader = req.Header
		// Respond with either a response body or a response code.
		if len(handler.respBody) > 0 {
			rw.Write([]byte(handler.respBody))
		} else {
			rw.WriteHeader(handler.respCode)
		}
		flusher := rw.(http.Flusher)
		flusher.Flush()
	}

	oth := ts.NewOutputTestHelper(ctrl)
	encoder := new(plugins.PayloadEncoder)
	encConfig := new(plugins.PayloadEncoderConfig)
	err := encoder.Init(encConfig)
	c.Expect(err, gs.IsNil)
	inChan := make(chan *pipeline.PipelinePack, 1)
	recycleChan := make(chan *pipeline.PipelinePack, 1)
	pack := pipeline.NewPipelinePack(recycleChan)
	pack.Message = pipeline_ts.GetTestMessage()

	c.Specify("An HttpOutput", func() {
		httpOutput := new(HttpOutput)
		config := httpOutput.ConfigStruct().(*HttpOutputConfig)

		c.Specify("barfs on bogus URLs", func() {
			config.Address = "one-two-three-four"
			err := httpOutput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("that is started", func() {
			server := httptest.NewServer(handler)
			defer server.Close()

			runOutput := func() {
				httpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				runWg.Done()
			}

			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			payload := "this is the payload"
			pack.Message.SetPayload(payload)
			oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(
				[]byte(payload), nil)
			config.Address = server.URL
			handler.respBody = "Response Body"

			c.Specify("makes http POST requests by default", func() {
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(reqMethod, gs.Equals, "POST")
			})

			c.Specify("makes http PUT requests", func() {
				config.Method = "put"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(reqMethod, gs.Equals, "PUT")
			})

			c.Specify("makes http GET requests", func() {
				config.Method = "get"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, "")
				c.Expect(reqMethod, gs.Equals, "GET")
			})

			c.Specify("correctly passes headers along", func() {
				config.Headers = http.Header{
					"One":  []string{"two", "three"},
					"Four": []string{"five", "six", "seven"},
				}
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"]))
				c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"]))
			})

			c.Specify("uses http auth when specified", func() {
				config.Username = "******"
				config.Password = "******"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				auth := reqHeader.Get("Authorization")
				c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue)
				decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:])
				c.Expect(err, gs.IsNil)
				c.Expect(string(decodedAuth), gs.Equals, "user:pass")
			})

			c.Specify("logs error responses", func() {
				handler.respBody = ""
				handler.respCode = 500
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)

				var errMsg string
				oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
					func(err error) {
						errMsg = err.Error()
					})
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(strings.HasPrefix(errMsg,
					"HTTP Error code returned: 500"), gs.IsTrue)
			})

			c.Specify("honors http timeout interval", func() {
				config.HttpTimeout = 1 // 1 millisecond
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)

				var errMsg string
				oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
					func(err error) {
						errMsg = err.Error()
					})
				delay = true
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(strings.Contains(errMsg, "use of closed network connection"),
					gs.IsTrue)
			})
		})
	})
}
Exemple #17
0
func CarbonOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := NewCarbonTestHelper(ctrl)
	pConfig := NewPipelineConfig(nil)

	// make test data
	const count = 5
	lines := make([]string, count)
	baseTime := time.Now().UTC().Add(-10 * time.Second)
	for i := 0; i < count; i++ {
		statName := fmt.Sprintf("stats.name.%d", i)
		statTime := baseTime.Add(time.Duration(i) * time.Second)
		lines[i] = fmt.Sprintf("%s %d %d", statName, i*2, statTime.Unix())
	}
	submit_data := strings.Join(lines, "\n")
	expected_data := submit_data + "\n"

	// a helper to make new test packs
	newpack := func() *PipelinePack {
		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Message.SetPayload(submit_data)
		pack.QueueCursor = "queuecursor"
		return pack
	}

	c.Specify("A CarbonOutput ", func() {
		inChan := make(chan *PipelinePack, 1)
		output := new(CarbonOutput)
		config := output.ConfigStruct().(*CarbonOutputConfig)
		pack := newpack()

		errChan := make(chan error, 1)
		connChan := make(chan net.Conn, 1)
		dataChan := make(chan string, count)

		startOutput := func(output *CarbonOutput, oth *CarbonTestHelper) {
			errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
		}

		// collectData waits for data to come in on the provided connection.
		// It "returns" data using channels, either with an error on the
		// errChan or data on the dataChan.
		collectData := func(conn net.Conn) {
			b := make([]byte, 10000)
			n, err := conn.Read(b)
			if err != nil {
				errChan <- err
				return
			}
			dataChan <- string(b[0:n])
		}

		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)

		var (
			conn net.Conn
			err  error
		)

		oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)

		c.Specify("using TCP", func() {
			var listener net.Listener

			config.Protocol = "tcp"
			listenerChan := make(chan net.Listener, 1)

			// startListener starts a TCP server waiting for data. It
			// "returns" data over channels. First it will *either* return an
			// error on the errChan *or* the listener on the listenerChan. If
			// listening succeeds it will wait for a connection attempt,
			// either returning an error on the errChan or the connection on
			// the connChan.
			startListener := func() {
				tcpaddr, err := net.ResolveTCPAddr("tcp", ":0")
				if err != nil {
					errChan <- err
					return
				}

				listener, err := net.ListenTCP("tcp", tcpaddr)
				if err != nil {
					errChan <- err
					return
				}
				listenerChan <- listener

				conn, err := listener.Accept()
				if err != nil {
					errChan <- err
					return
				}
				connChan <- conn
			}

			go startListener()
			select {
			case err = <-errChan:
				// If we get an error herer it won't be nil, but we use
				// c.Assume to trigger the test failure.
				c.Assume(err, gs.IsNil)
			case listener = <-listenerChan:
				defer listener.Close()
			}

			c.Specify("writes to the network", func() {

				config.Address = fmt.Sprintf("127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port)
				err = output.Init(config)
				c.Assume(err, gs.IsNil)
				inChan <- pack
				go startOutput(output, oth)

				select {
				case err = <-errChan:
					// If we get an error herer it won't be nil, but we use
					// c.Assume to trigger the test failure.
					c.Assume(err, gs.IsNil)
				case conn = <-connChan:
					defer conn.Close()
				}

				go collectData(conn)

				select {
				case err = <-errChan:
					// If we get an error herer it won't be nil, but we use
					// c.Assume to trigger the test failure.
					c.Assume(err, gs.IsNil)
				case data := <-dataChan:
					c.Expect(data, gs.Equals, expected_data)
				}

				close(inChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("using UDP", func() {
			config.Protocol = "udp"

			// startListener starts a UDP server waiting for data. It
			// "returns" data over channels. It will either return a net.Conn
			// on the connChan or an error on the errChan.
			startListener := func() {
				udpAddr, err := net.ResolveUDPAddr("udp", ":0")
				if err != nil {
					errChan <- err
					return
				}
				conn, err := net.ListenUDP("udp", udpAddr)
				if err != nil {
					errChan <- err
					return
				}
				connChan <- conn
			}

			go startListener()
			select {
			case err = <-errChan:
				c.Assume(err, gs.IsNil)
			case conn = <-connChan:
				defer conn.Close()
			}

			c.Specify("writes to the network", func() {
				config.Address = fmt.Sprintf("127.0.0.1:%d", conn.LocalAddr().(*net.UDPAddr).Port)
				err = output.Init(config)
				c.Assume(err, gs.IsNil)
				inChan <- pack

				go startOutput(output, oth)
				go collectData(conn)

				select {
				case err = <-errChan:
					// If we get an error herer it won't be nil, but we use
					// c.Assume to trigger the test failure.
					c.Assume(err, gs.IsNil)
				case data := <-dataChan:
					c.Expect(data, gs.Equals, expected_data)
				}

				close(inChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})

			c.Specify("splits packets that are too long", func() {
				config.Address = fmt.Sprintf("127.0.0.1:%d", conn.LocalAddr().(*net.UDPAddr).Port)
				err = output.Init(config)
				c.Assume(err, gs.IsNil)
				output.bufSplitSize = 1 // Forces a separate packet for each stat.

				inChan <- pack

				go startOutput(output, oth)

				for i := 0; i < count; i++ {
					go collectData(conn)

					select {
					case err = <-errChan:
						// If we get an error herer it won't be nil, but we use
						// c.Assume to trigger the test failure.
						c.Assume(err, gs.IsNil)
					case data := <-dataChan:
						c.Expect(data, gs.Equals, lines[i]+"\n")
					}
				}

				close(inChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})
	})
}
Exemple #18
0
func SmtpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	encoder := new(plugins.PayloadEncoder)
	econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
	econfig.AppendNewlines = false
	encoder.Init(econfig)

	c.Specify("A SmtpOutput", func() {
		smtpOutput := new(SmtpOutput)

		config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
		config.SendTo = []string{"root"}

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)
		runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
		runnerName.Return("SmtpOutput")
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
		encCall := oth.MockOutputRunner.EXPECT().Encode(pack)

		c.Specify("send email payload message", func() {
			err := smtpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			smtpOutput.sendFunction = testSendMail

			outStr := "Write me out to the network"
			pack.Message.SetPayload(outStr)
			encCall.Return(encoder.Encode(pack))
			wg.Add(1)
			go func() {
				smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
			inChan <- pack
			close(inChan)
			wg.Wait()
		})
	})

	c.Specify("SmtpOutput Message Body Encoding", func() {
		smtpOutput := new(SmtpOutput)
		chars := "123456789012345678901234567890123456789012345678901234567"
		charsE := "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3"
		examples := [][]string{
			{"Hello", "SGVsbG8="},
			{chars, charsE},
			{chars + chars, charsE + "\r\n" + charsE},
			{chars + chars + "Hello", charsE + "\r\n" + charsE + "\r\n" + "SGVsbG8="},
			{"", ""},
			{"1", "MQ=="},
		}
		for _, example := range examples {
			smtpOutput.encodeFullMsg([]byte(example[0]))
			c.Expect(string(smtpOutput.fullMsg), gs.Equals, example[1])
		}
	})

	// // Use this test with a real server
	// c.Specify("Real SmtpOutput output", func() {
	// 	smtpOutput := new(SmtpOutput)

	// 	config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
	// 	config.SendTo = []string{"root"}

	// 	msg := pipeline_ts.GetTestMessage()
	// 	pack := NewPipelinePack(pConfig.InputRecycleChan())
	// 	pack.Message = msg
	// 	pack.Decoded = true
	// 	inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
	// 	inChanCall.Return(inChan)
	// 	runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
	// 	runnerName.Return("SmtpOutput")
	// 	oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
	// 	encCall := oth.MockOutputRunner.EXPECT().Encode(pack)

	// 	c.Specify("send a real email essage", func() {

	// 		err := smtpOutput.Init(config)
	// 		c.Assume(err, gs.IsNil)

	// 		outStr := "Write me out to the network"
	// 		pack.Message.SetPayload(outStr)
	// 		encCall.Return(encoder.Encode(pack))
	// 		go func() {
	// 			wg.Add(1)
	// 			smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
	// 			wg.Done()
	// 		}()
	// 		inChan <- pack
	// 		time.Sleep(1000) // allow time for the message output
	// 		close(inChan)
	// 		wg.Wait()
	// // manually check the mail
	//	})
	// })
}
Exemple #19
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	config := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: config}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}
		lsiConfig.Decoder = "decoder-name"

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)
			mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
				// Expect calls to get decoder and decode each message. Since the
				// decoding is a no-op, the message payload will be the log file
				// line, unchanged.
				pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
					"-"+lsiConfig.Decoder)
				pbcall.Return(mockDecoderRunner, true)

				decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
				decodeCall.Return(ith.DecodeChan)

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				d, _ := time.ParseDuration("5s")
				timeout := time.After(d)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case <-ith.DecodeChan:
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Exemple #20
0
func FileOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	c.Specify("A FileOutput", func() {
		fileOutput := new(FileOutput)

		tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
		tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
			tmpFileName)
		config := fileOutput.ConfigStruct().(*FileOutputConfig)
		config.Path = tmpFilePath

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		toString := func(outData interface{}) string {
			return string(*(outData.(*[]byte)))
		}

		c.Specify("correctly formats text output", func() {
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 20)

			c.Specify("by default", func() {
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
			})

			c.Specify("w/ a prepended timestamp when specified", func() {
				fileOutput.prefix_ts = true
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				payload := *msg.Payload
				c.Expect(strContents, pipeline_ts.StringContains, payload)
				c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
			})

			c.Specify("even when payload is nil", func() {
				pack.Message.Payload = nil
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				strContents := toString(&outData)
				c.Expect(strContents, gs.Equals, "\n")
			})

			c.Specify("payload is nil and with a timestamp", func() {
				pack.Message.Payload = nil
				fileOutput.prefix_ts = true
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				c.Expect(strings.HasPrefix(strContents, todayStr), gs.IsTrue)
				c.Expect(strings.HasSuffix(strContents, " \n"), gs.IsTrue)
			})
		})

		c.Specify("correctly formats JSON output", func() {
			config.Format = "json"
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 200)

			c.Specify("when specified", func() {
				fileOutput.handleMessage(pack, &outData)
				msgJson, err := json.Marshal(pack.Message)
				c.Assume(err, gs.IsNil)
				c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
			})

			c.Specify("and with a timestamp", func() {
				fileOutput.prefix_ts = true
				fileOutput.handleMessage(pack, &outData)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				msgJson, err := json.Marshal(pack.Message)
				c.Assume(err, gs.IsNil)
				c.Expect(strContents, pipeline_ts.StringContains, string(msgJson)+"\n")
				c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
			})
		})

		c.Specify("correctly formats protocol buffer stream output", func() {
			config.Format = "protobufstream"
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 200)

			c.Specify("when specified and timestamp ignored", func() {
				fileOutput.prefix_ts = true
				err := fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
				c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
			})
		})

		c.Specify("processes incoming messages", func() {
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			// Save for comparison.
			payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())

			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			wg.Add(1)
			go fileOutput.receiver(oth.MockOutputRunner, &wg)
			inChan <- pack
			close(inChan)
			outBatch := <-fileOutput.batchChan
			wg.Wait()
			c.Expect(string(outBatch), gs.Equals, payload)
		})

		c.Specify("Init halts if basedirectory is not writable", func() {
			tmpdir := filepath.Join(os.TempDir(), "tmpdir")
			err := os.MkdirAll(tmpdir, 0400)
			c.Assume(err, gs.IsNil)
			config.Path = tmpdir
			err = fileOutput.Init(config)
			c.Assume(err, gs.Not(gs.IsNil))
		})

		c.Specify("commits to a file", func() {
			outStr := "Write me out to the log file"
			outBytes := []byte(outStr)

			c.Specify("with default settings", func() {
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				contents, err := ioutil.ReadAll(tmpFile)
				c.Assume(err, gs.IsNil)
				c.Expect(string(contents), gs.Equals, outStr)
			})

			c.Specify("with different Perm settings", func() {
				config.Perm = "600"
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				fileInfo, err := tmpFile.Stat()
				c.Assume(err, gs.IsNil)
				fileMode := fileInfo.Mode()
				if runtime.GOOS == "windows" {
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
				} else {
					// 7 consecutive dashes implies no perms for group or other
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
				}
			})
		})

		c.Specify("honors folder_perm setting", func() {
			config.FolderPerm = "750"
			config.Path = filepath.Join(tmpFilePath, "subfile")
			err := fileOutput.Init(config)
			defer os.Remove(config.Path)
			c.Assume(err, gs.IsNil)

			fi, err := os.Stat(tmpFilePath)
			c.Expect(fi.IsDir(), gs.IsTrue)
			c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
		})

		c.Specify("that starts receiving w/ a flush interval", func() {
			config.FlushInterval = 100000000 // We'll trigger the timer manually.
			inChan := make(chan *PipelinePack)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			timerChan := make(chan time.Time)

			msg2 := pipeline_ts.GetTestMessage()
			pack2 := NewPipelinePack(pConfig.InputRecycleChan())
			pack2.Message = msg2

			recvWithConfig := func(config *FileOutputConfig) {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)
				wg.Add(1)
				go fileOutput.receiver(oth.MockOutputRunner, &wg)
				runtime.Gosched() // Yield so we can overwrite the timerChan.
				fileOutput.timerChan = timerChan
			}

			cleanUp := func() {
				close(inChan)
				wg.Done()
			}

			c.Specify("honors flush interval", func() {
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack
				select {
				case _ = <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}
				timerChan <- time.Now()
				select {
				case _ = <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval AND flush count", func() {
				config.FlushCount = 2
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				timerChan <- time.Now()
				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				inChan <- pack2
				runtime.Gosched()
				select {
				case <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval OR flush count", func() {
				config.FlushCount = 2
				config.FlushOperator = "OR"
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				c.Specify("when interval triggers first", func() {
					timerChan <- time.Now()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})

				c.Specify("when count triggers first", func() {
					inChan <- pack2
					runtime.Gosched()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})
			})
		})
	})
}
Exemple #21
0
func UdpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	c.Specify("A UdpInput", func() {
		udpInput := UdpInput{}
		config := &UdpInputConfig{}

		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))
		hbytes, _ := proto.Marshal(header)
		buf := encodeMessage(hbytes, mbytes)

		bytesChan := make(chan []byte, 1)

		ith.MockInputRunner.EXPECT().Name().Return("mock_name")
		ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
		ith.MockSplitterRunner.EXPECT().GetRemainingData().AnyTimes()
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

		splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
			nil).AnyTimes()
		splitCall.Do(func(conn net.Conn, del Deliverer) {
			recd := make([]byte, 65536)
			n, _ := conn.Read(recd)
			recd = recd[:n]
			bytesChan <- recd
		})

		c.Specify("using a udp address", func() {
			ith.AddrStr = "localhost:55565"
			ith.ResolvedAddrStr = "127.0.0.1:55565"
			config.Net = "udp"
			config.Address = ith.AddrStr

			err := udpInput.Init(config)
			c.Assume(err, gs.IsNil)
			realListener := (udpInput.listener).(*net.UDPConn)
			c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)

			c.Specify("passes the connection to SplitStream", func() {
				go udpInput.Run(ith.MockInputRunner, ith.MockHelper)

				conn, err := net.Dial("udp", ith.AddrStr)
				c.Assume(err, gs.IsNil)
				_, err = conn.Write(buf)
				c.Assume(err, gs.IsNil)
				conn.Close()

				recd := <-bytesChan
				c.Expect(string(recd), gs.Equals, string(buf))
				udpInput.Stop()
			})
		})

		if runtime.GOOS != "windows" {
			c.Specify("using a unix datagram socket", func() {
				tmpDir, err := ioutil.TempDir("", "heka-socket")
				c.Assume(err, gs.IsNil)
				unixPath := filepath.Join(tmpDir, "unixgram-socket")
				ith.AddrStr = unixPath
				config.Net = "unixgram"
				config.Address = ith.AddrStr

				err = udpInput.Init(config)
				c.Assume(err, gs.IsNil)
				realListener := (udpInput.listener).(*net.UnixConn)
				c.Expect(realListener.LocalAddr().String(), gs.Equals, unixPath)

				c.Specify("passes the socket to SplitStream", func() {
					go udpInput.Run(ith.MockInputRunner, ith.MockHelper)

					unixAddr, err := net.ResolveUnixAddr("unixgram", unixPath)
					c.Assume(err, gs.IsNil)
					conn, err := net.DialUnix("unixgram", nil, unixAddr)
					c.Assume(err, gs.IsNil)
					_, err = conn.Write(buf)
					c.Assume(err, gs.IsNil)
					conn.Close()

					recd := <-bytesChan
					c.Expect(string(recd), gs.Equals, string(buf))
					udpInput.Stop()
				})
			})
		}
	})
}
Exemple #22
0
func OutputRunnerSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mockHelper := NewMockPluginHelper(ctrl)

	c.Specify("A runner", func() {
		stopoutputTimes = 0
		pc := NewPipelineConfig(nil)
		pluginGlobals := new(PluginGlobals)

		c.Specify("restarts a plugin on the first time only", func() {
			pluginGlobals.Retries = RetryOptions{
				MaxDelay:   "1us",
				Delay:      "1us",
				MaxJitter:  "1us",
				MaxRetries: 1,
			}
			pw := NewPluginWrapper("stoppingOutput", pc)
			pw.ConfigCreator = func() interface{} { return nil }
			pw.PluginCreator = func() interface{} { return new(StoppingOutput) }
			output := new(StoppingOutput)
			pc.outputWrappers = make(map[string]*PluginWrapper)
			pc.outputWrappers["stoppingOutput"] = pw
			oRunner := NewFORunner("stoppingOutput", output, pluginGlobals, 10)
			var wg sync.WaitGroup
			cfgCall := mockHelper.EXPECT().PipelineConfig()
			cfgCall.Return(pc)
			wg.Add(1)
			oRunner.Start(mockHelper, &wg) // no panic => success
			wg.Wait()
			c.Expect(stopoutputTimes, gs.Equals, 2)
		})

		c.Specify("restarts plugin and resumes feeding it", func() {
			pluginGlobals.Retries = RetryOptions{
				MaxDelay:   "1us",
				Delay:      "1us",
				MaxJitter:  "1us",
				MaxRetries: 4,
			}
			pw := NewPluginWrapper("stoppingresumeOutput", pc)
			pw.ConfigCreator = func() interface{} { return nil }
			pw.PluginCreator = func() interface{} { return new(StopResumeOutput) }
			output := new(StopResumeOutput)
			pc.outputWrappers = make(map[string]*PluginWrapper)
			pc.outputWrappers["stoppingresumeOutput"] = pw
			oRunner := NewFORunner("stoppingresumeOutput", output, pluginGlobals, 10)
			var wg sync.WaitGroup
			cfgCall := mockHelper.EXPECT().PipelineConfig()
			cfgCall.Return(pc)
			wg.Add(1)
			oRunner.Start(mockHelper, &wg) // no panic => success
			wg.Wait()
			c.Expect(stopresumerunTimes, gs.Equals, 3)
			c.Expect(len(stopresumeHolder), gs.Equals, 2)
			c.Expect(stopresumeHolder[1], gs.Equals, "woot")
			c.Expect(oRunner.retainPack, gs.IsNil)
		})

		c.Specify("can exit without causing shutdown", func() {
			pluginGlobals.Retries = RetryOptions{MaxRetries: 0}
			pw := NewPluginWrapper("stoppingOutput", pc)
			pw.ConfigCreator = func() interface{} { return nil }
			pw.PluginCreator = func() interface{} { return new(StoppingOutput) }
			output := new(StoppingOutput)
			pc.outputWrappers = make(map[string]*PluginWrapper)
			pc.outputWrappers["stoppingOutput"] = pw
			oRunner := NewFORunner("stoppingOutput", output, pluginGlobals, 10)
			oRunner.canExit = true

			// This pack is for the sending of the terminated message
			pack := NewPipelinePack(pc.injectRecycleChan)
			pc.injectRecycleChan <- pack

			// Feed in a pack to the input so we can verify its been recycled
			// after stopping (no leaks)
			pack = NewPipelinePack(pc.inputRecycleChan)
			oRunner.inChan <- pack

			// This is code to emulate the router removing the Output, and
			// closing up the outputs inChan channel
			go func() {
				<-pc.Router().RemoveOutputMatcher()
				// We don't close the matcher inChan because its not
				// instantiated in the tests
				close(oRunner.inChan)
			}()
			var wg sync.WaitGroup
			cfgCall := mockHelper.EXPECT().PipelineConfig()
			cfgCall.Return(pc)

			wg.Add(1)
			oRunner.Start(mockHelper, &wg)
			wg.Wait()
			c.Expect(stopoutputTimes, gs.Equals, 1)
			p := <-pc.router.inChan
			c.Expect(p.Message.GetType(), gs.Equals, "heka.terminated")
			// This should be 1 because the inChan should be flushed
			// and packs should not be leaked
			c.Expect(len(pc.inputRecycleChan), gs.Equals, 1)
		})

		c.Specify("encodes a message", func() {
			output := new(StoppingOutput)
			or := NewFORunner("test", output, pluginGlobals, 10)
			or.encoder = new(_payloadEncoder)
			_pack.Message = ts.GetTestMessage()
			payload := "Test Payload"

			c.Specify("without framing", func() {
				result, err := or.Encode(_pack)
				c.Expect(err, gs.IsNil)
				c.Expect(string(result), gs.Equals, payload)
			})

			c.Specify("with framing", func() {
				or.SetUseFraming(true)
				result, err := or.Encode(_pack)
				c.Expect(err, gs.IsNil)

				i := bytes.IndexByte(result, message.UNIT_SEPARATOR)
				c.Expect(i > 3, gs.IsTrue, -1)
				c.Expect(string(result[i+1:]), gs.Equals, payload)
				header := new(message.Header)
				ok := DecodeHeader(result[2:i+1], header)
				c.Expect(ok, gs.IsTrue)
				c.Expect(header.GetMessageLength(), gs.Equals, uint32(len(payload)))
			})
		})
	})
}
Exemple #23
0
func FileOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
	tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)

	defer func() {
		ctrl.Finish()
		os.Remove(tmpFilePath)
	}()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	c.Specify("A FileOutput", func() {
		fileOutput := new(FileOutput)
		encoder := new(plugins.PayloadEncoder)
		encoder.Init(encoder.ConfigStruct())

		config := fileOutput.ConfigStruct().(*FileOutputConfig)
		config.Path = tmpFilePath

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("w/ ProtobufEncoder", func() {
			encoder := new(ProtobufEncoder)
			encoder.Init(nil)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)

			c.Specify("uses framing", func() {
				oth.MockOutputRunner.EXPECT().SetUseFraming(true)
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)
				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				wg.Add(1)
				go func() {
					err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()
				close(inChan)
				wg.Wait()
			})

			c.Specify("but not if config says not to", func() {
				useFraming := false
				config.UseFraming = &useFraming
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)
				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				wg.Add(1)
				go func() {
					err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()
				close(inChan)
				wg.Wait()
				// We should fail if SetUseFraming is called since we didn't
				// EXPECT it.

			})
		})

		c.Specify("processes incoming messages", func() {
			err := fileOutput.Init(config)
			c.Assume(err, gs.IsNil)
			fileOutput.file.Close()
			// Save for comparison.
			payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())

			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			wg.Add(1)
			go fileOutput.receiver(oth.MockOutputRunner, &wg)
			inChan <- pack
			close(inChan)
			outBatch := <-fileOutput.batchChan
			wg.Wait()
			c.Expect(string(outBatch), gs.Equals, payload)
		})

		c.Specify("commits to a file", func() {
			outStr := "Write me out to the log file"
			outBytes := []byte(outStr)

			c.Specify("with default settings", func() {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				contents, err := ioutil.ReadAll(tmpFile)
				c.Assume(err, gs.IsNil)
				c.Expect(string(contents), gs.Equals, outStr)
			})

			c.Specify("with different Perm settings", func() {
				config.Perm = "600"
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				fileInfo, err := tmpFile.Stat()
				c.Assume(err, gs.IsNil)
				fileMode := fileInfo.Mode()
				if runtime.GOOS == "windows" {
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
				} else {
					// 7 consecutive dashes implies no perms for group or other
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
				}
			})
		})

		if runtime.GOOS != "windows" {
			c.Specify("Init halts if basedirectory is not writable", func() {
				tmpdir := filepath.Join(os.TempDir(), "tmpdir")
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				config.Path = filepath.Join(tmpdir, "out.txt")
				err = fileOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
				os.RemoveAll(tmpdir)
			})

			c.Specify("honors folder_perm setting", func() {
				config.FolderPerm = "750"
				subdir := filepath.Join(os.TempDir(), "subdir")
				config.Path = filepath.Join(subdir, "out.txt")
				err := fileOutput.Init(config)
				defer os.RemoveAll(subdir)
				c.Assume(err, gs.IsNil)

				fi, err := os.Stat(subdir)
				c.Expect(fi.IsDir(), gs.IsTrue)
				c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
			})
		}

		c.Specify("that starts receiving w/ a flush interval", func() {
			config.FlushInterval = 100000000 // We'll trigger the timer manually.
			inChan := make(chan *PipelinePack)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			timerChan := make(chan time.Time)

			msg2 := pipeline_ts.GetTestMessage()
			pack2 := NewPipelinePack(pConfig.InputRecycleChan())
			pack2.Message = msg2

			recvWithConfig := func(config *FileOutputConfig) {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)
				wg.Add(1)
				go fileOutput.receiver(oth.MockOutputRunner, &wg)
				runtime.Gosched() // Yield so we can overwrite the timerChan.
				fileOutput.timerChan = timerChan
			}

			cleanUp := func() {
				close(inChan)
				fileOutput.file.Close()
				wg.Done()
			}

			c.Specify("honors flush interval", func() {
				oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack
				select {
				case _ = <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}
				timerChan <- time.Now()
				select {
				case _ = <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval AND flush count", func() {
				oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
				oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
				config.FlushCount = 2
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				timerChan <- time.Now()
				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				inChan <- pack2
				runtime.Gosched()
				select {
				case <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval OR flush count", func() {
				oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
				config.FlushCount = 2
				config.FlushOperator = "OR"
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				c.Specify("when interval triggers first", func() {
					timerChan <- time.Now()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})

				c.Specify("when count triggers first", func() {
					out, err := encoder.Encode(pack2)
					oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
					inChan <- pack2
					runtime.Gosched()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})
			})
		})
	})
}
Exemple #24
0
func TcpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A TcpInput", func() {
		tcpInput := &TcpInput{}
		config := &TcpInputConfig{
			Net:     "tcp",
			Address: ith.AddrStr,
		}

		bytesChan := make(chan []byte, 1)
		errChan := make(chan error, 1)
		var srDoneWG sync.WaitGroup

		startServer := func() {
			srDoneWG.Add(1)
			ith.MockInputRunner.EXPECT().Name().Return("mock_name")
			ith.MockInputRunner.EXPECT().NewDeliverer(gomock.Any()).Return(ith.MockDeliverer)
			ith.MockDeliverer.EXPECT().Done()
			ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
				ith.MockSplitterRunner)
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			ith.MockSplitterRunner.EXPECT().Done().Do(func() {
				srDoneWG.Done()
			})

			// splitCall gets called twice. The first time it returns nil, the
			// second time it returns io.EOF.
			splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
				ith.MockDeliverer).AnyTimes()
			splitCall.Do(func(conn net.Conn, del Deliverer) {
				recd, _ := ioutil.ReadAll(conn)
				bytesChan <- recd
				splitCall.Return(io.EOF)
			})

			err := tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			errChan <- err
		}

		c.Specify("not using TLS", func() {
			err := tcpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(tcpInput.listener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)

			c.Specify("accepts connections and passes them to the splitter", func() {
				go startServer()
				data := []byte("THIS IS THE DATA")

				outConn, err := net.Dial("tcp", ith.AddrStr)
				c.Assume(err, gs.IsNil)
				_, err = outConn.Write(data)
				c.Expect(err, gs.IsNil)
				outConn.Close()

				recd := <-bytesChan

				c.Expect(err, gs.IsNil)
				c.Expect(string(recd), gs.Equals, string(data))

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})
		})

		c.Specify("using TLS", func() {
			config.UseTls = true

			c.Specify("fails to init w/ missing key or cert file", func() {
				config.Tls = TlsConfig{}
				err := tcpInput.Init(config)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"TLS config requires both cert_file and key_file value.")
			})

			config.Tls = TlsConfig{
				CertFile: "./testsupport/cert.pem",
				KeyFile:  "./testsupport/key.pem",
			}

			c.Specify("accepts connections and passes them to the splitter", func() {
				err := tcpInput.Init(config)
				c.Expect(err, gs.IsNil)

				go startServer()
				data := []byte("This is a test.")

				clientConfig := new(tls.Config)
				clientConfig.InsecureSkipVerify = true
				outConn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
				c.Assume(err, gs.IsNil)
				outConn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
				n, err := outConn.Write(data)
				c.Expect(err, gs.IsNil)
				c.Expect(n, gs.Equals, len(data))
				outConn.Close()

				recd := <-bytesChan

				c.Expect(err, gs.IsNil)
				c.Expect(string(recd), gs.Equals, string(data))

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})

			c.Specify("doesn't accept connections below specified min TLS version", func() {
				config.Tls.MinVersion = "TLS12"
				err := tcpInput.Init(config)
				c.Expect(err, gs.IsNil)

				go startServer()

				clientConfig := &tls.Config{
					InsecureSkipVerify: true,
					MaxVersion:         tls.VersionTLS11,
				}
				conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
				c.Expect(conn, gs.IsNil)
				c.Expect(err, gs.Not(gs.IsNil))

				<-bytesChan

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})
		})
	})
}
func ProcessDirectoryInputSpec(c gs.Context) {

	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	ith.PackSupply <- ith.Pack

	err := pConfig.RegisterDefault("NullSplitter")
	c.Assume(err, gs.IsNil)

	c.Specify("A ProcessDirectoryInput", func() {
		pdiInput := ProcessDirectoryInput{}
		pdiInput.SetPipelineConfig(pConfig)

		config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig)
		workingDir, err := os.Getwd()
		c.Assume(err, gs.IsNil)
		config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes")

		// `Ticker` is the last thing called during the setup part of the
		// input's `Run` method, so it triggers a waitgroup that tests can
		// wait on when they need to ensure initialization has finished.
		var started sync.WaitGroup
		started.Add(1)
		tickChan := make(chan time.Time, 1)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do(
			func() {
				started.Done()
			})

		// Similarly we use a waitgroup to signal when LogMessage has been
		// called to know when reloads have completed. Warning: If you call
		// expectLogMessage with a msg that is never passed to LogMessage and
		// then you call loaded.Wait() then your test will hang and never
		// complete.
		var loaded sync.WaitGroup
		expectLogMessage := func(msg string) {
			loaded.Add(1)
			ith.MockInputRunner.EXPECT().LogMessage(msg).Do(
				func(msg string) {
					loaded.Done()
				})
		}

		// Same name => same content.
		paths := []string{
			filepath.Join(config.ProcessDir, "100", "h0.toml"),
			filepath.Join(config.ProcessDir, "100", "h1.toml"),
			filepath.Join(config.ProcessDir, "200", "h0.toml"),
			filepath.Join(config.ProcessDir, "300", "h1.toml"),
		}

		copyFile := func(src, dest string) {
			inFile, err := os.Open(src)
			c.Assume(err, gs.IsNil)
			outFile, err := os.Create(dest)
			c.Assume(err, gs.IsNil)
			_, err = io.Copy(outFile, inFile)
			c.Assume(err, gs.IsNil)
			inFile.Close()
			outFile.Close()
		}

		err = pdiInput.Init(config)
		c.Expect(err, gs.IsNil)

		for _, p := range paths {
			expectLogMessage("Added: " + p)
		}
		go pdiInput.Run(ith.MockInputRunner, ith.MockHelper)
		defer func() {
			pdiInput.Stop()
			for _, entry := range pdiInput.inputs {
				entry.ir.Input().Stop()
			}
		}()
		started.Wait()

		c.Specify("loads scheduled jobs", func() {
			pathIndex := func(name string) (i int) {
				var p string
				for i, p = range paths {
					if name == p {
						return
					}
				}
				return -1
			}

			for name, entry := range pdiInput.inputs {
				i := pathIndex(name)
				// Make sure each file path got registered.
				c.Expect(i, gs.Not(gs.Equals), -1)
				dirName := filepath.Base(filepath.Dir(name))
				dirInt, err := strconv.Atoi(dirName)
				c.Expect(err, gs.IsNil)
				// And that the ticker interval was read correctly.
				c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval)
			}
		})

		c.Specify("discovers and adds a new job", func() {
			// Copy one of the files to register a new process.
			newPath := filepath.Join(config.ProcessDir, "300", "h0.toml")
			copyFile(paths[0], newPath)
			defer func() {
				err := os.Remove(newPath)
				c.Assume(err, gs.IsNil)
			}()

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Added: " + newPath)
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure our plugin was loaded.
			c.Expect(len(pdiInput.inputs), gs.Equals, 5)
			newEntry, ok := pdiInput.inputs[newPath]
			c.Expect(ok, gs.IsTrue)
			c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300))
		})

		c.Specify("removes a deleted job", func() {
			err := os.Remove(paths[3])
			c.Assume(err, gs.IsNil)
			defer func() {
				copyFile(paths[1], paths[3])
			}()

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Removed: " + paths[3])
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure our plugin was deleted.
			c.Expect(len(pdiInput.inputs), gs.Equals, 3)
		})

		c.Specify("notices a changed job", func() {
			// Overwrite one job w/ a slightly different one.
			copyFile(paths[0], paths[3])
			defer copyFile(paths[1], paths[3])

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Removed: " + paths[3])
			expectLogMessage("Added: " + paths[3])
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure the new config was loaded.
			c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals,
				"hello world\n")
		})
	})

}
Exemple #26
0
func HekaFramingSpec(c gs.Context) {
	c.Specify("A HekaFramingSplitter", func() {
		splitter := &HekaFramingSplitter{}
		config := splitter.ConfigStruct().(*HekaFramingSplitterConfig)
		sRunner := makeSplitterRunner("HekaFramingSplitter", splitter)

		c.Specify("splits records", func() {
			b := []byte("\x1e\x02\x08\x3e\x1f\x0a\x10\x90\x1d\x56\x27\xec\x49\x4c\x8f\xba\x8e\x84\x9b\xaa\xf7\xa6\xf6\x10\xa6\x97\x8a\x8f\xb6\xc1\xae\x8e\x13\x1a\x09\x68\x65\x6b\x61\x62\x65\x6e\x63\x68\x28\x06\x3a\x03\x30\x2e\x38\x40\xbf\xe5\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x1e\x02\x08\x3e\x1f\x0a\x10\x90\x1d\x56\x27\xec\x49\x4c\x8f\xba\x8e\x84\x9b\xaa\xf7\xa6\xf6\x10\xa6\x97\x8a\x8f\xb6\xc1\xae\x8e\x13\x1a\x09\x68\x65\x6b\x61\x62\x65\x6e\x63\x68\x28\x06\x3a\x03\x30\x2e\x38\x40\xbf\xe5\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30BOGUS\x1e\x02\x08\x3e\x1f\x0a\x10\x90\x1d\x56\x27\xec\x49\x4c\x8f\xba\x8e\x84\x9b\xaa\xf7\xa6\xf6\x10\xa6\x97\x8a\x8f\xb6\xc1\xae\x8e\x13\x1a\x09\x68\x65\x6b\x61\x62\x65\x6e\x63\x68\x28\x06\x3a\x03\x30\x2e\x38\x40\xbf\xe5\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30BOGUS\x1e\x02\x08")
			reader := bytes.NewReader(b)
			err := splitter.Init(config)
			c.Assume(err, gs.IsNil)

			n, record, err := sRunner.GetRecordFromStream(reader)
			c.Expect(n, gs.Equals, 67)
			c.Expect(err, gs.IsNil)
			c.Expect(string(record), gs.Equals, string(b[:67]))
			n, record, err = sRunner.GetRecordFromStream(reader)
			c.Expect(n, gs.Equals, 67)
			c.Expect(err, gs.IsNil)
			c.Expect(string(record), gs.Equals, string(b[67:134]))
			n, record, err = sRunner.GetRecordFromStream(reader)
			c.Expect(n, gs.Equals, 72) // skips the invalid 'BOGUS' data
			c.Expect(err, gs.IsNil)
			c.Expect(string(record), gs.Equals, string(b[139:206]))
			n, record, err = sRunner.GetRecordFromStream(reader) // trigger the need to read more data
			c.Expect(n, gs.Equals, 5)
			c.Expect(err, gs.IsNil)
			c.Expect(len(record), gs.Equals, 0)
			n, record, err = sRunner.GetRecordFromStream(reader) // hit the EOF
			c.Expect(n, gs.Equals, 0)
			c.Expect(err, gs.Equals, io.EOF)
			c.Expect(len(record), gs.Equals, 0)
		})

		c.Specify("correctly handles invalid header (no unit separator)", func() {
			b := []byte("\x1e\x02\x08\x3e\xff\x1e\x02\x08\x3e\x1f\x0a\x10\x90\x1d\x56\x27\xec\x49\x4c\x8f\xba\x8e\x84\x9b\xaa\xf7\xa6\xf6\x10\xa6\x97\x8a\x8f\xb6\xc1\xae\x8e\x13\x1a\x09\x68\x65\x6b\x61\x62\x65\x6e\x63\x68\x28\x06\x3a\x03\x30\x2e\x38\x40\xbf\xe5\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30")
			reader := bytes.NewReader(b)
			err := splitter.Init(config)
			c.Assume(err, gs.IsNil)

			n, record, err := sRunner.GetRecordFromStream(reader)
			c.Expect(n, gs.Equals, 72)
			c.Expect(err, gs.IsNil)
			c.Expect(string(record), gs.Equals, string(b[5:]))
		})

		c.Specify("using authentication", func() {
			key := "testkey"
			config.Signers = map[string]Signer{"test_1": {key}}
			signer := "test"
			recycleChan := make(chan *PipelinePack, 1)
			pack := NewPipelinePack(recycleChan)
			msg := ts.GetTestMessage()
			mbytes, _ := proto.Marshal(msg)
			header := &message.Header{}
			header.SetMessageLength(uint32(len(mbytes)))

			c.Specify("authenticates MD5 signed message", func() {
				err := splitter.Init(config)
				c.Assume(err, gs.IsNil)

				header.SetHmacHashFunction(message.Header_MD5)
				header.SetHmacSigner(signer)
				header.SetHmacKeyVersion(uint32(1))
				hm := hmac.New(md5.New, []byte(key))
				hm.Write(mbytes)
				header.SetHmac(hm.Sum(nil))
				hbytes, _ := proto.Marshal(header)

				framed := encodeMessage(hbytes, mbytes)
				unframed := splitter.UnframeRecord(framed, pack)
				c.Expect(pack.Signer, gs.Equals, "test")
				c.Expect(string(unframed), gs.Equals, string(mbytes))
			})

			c.Specify("authenticates SHA1 signed message", func() {
				err := splitter.Init(config)
				c.Assume(err, gs.IsNil)

				header.SetHmacHashFunction(message.Header_SHA1)
				header.SetHmacSigner(signer)
				header.SetHmacKeyVersion(uint32(1))
				hm := hmac.New(sha1.New, []byte(key))
				hm.Write(mbytes)
				header.SetHmac(hm.Sum(nil))
				hbytes, _ := proto.Marshal(header)

				framed := encodeMessage(hbytes, mbytes)
				unframed := splitter.UnframeRecord(framed, pack)
				c.Expect(pack.Signer, gs.Equals, "test")
				c.Expect(string(unframed), gs.Equals, string(mbytes))
			})

			c.Specify("doesn't auth signed message with expired key", func() {
				err := splitter.Init(config)
				c.Assume(err, gs.IsNil)

				header.SetHmacHashFunction(message.Header_MD5)
				header.SetHmacSigner(signer)
				header.SetHmacKeyVersion(uint32(11)) // non-existent key version
				hm := hmac.New(md5.New, []byte(key))
				hm.Write(mbytes)
				header.SetHmac(hm.Sum(nil))
				hbytes, _ := proto.Marshal(header)

				framed := encodeMessage(hbytes, mbytes)
				unframed := splitter.UnframeRecord(framed, pack)
				c.Expect(pack.Signer, gs.Equals, "")
				// The function returns nil, and `unframed == nil` evaluates
				// to true, but `gs.IsNil` doesn't work here.
				c.Expect(string(unframed), gs.Equals, "")
			})

			c.Specify("doesn't auth signed message with incorrect hmac", func() {
				err := splitter.Init(config)
				c.Assume(err, gs.IsNil)

				header.SetHmacHashFunction(message.Header_MD5)
				header.SetHmacSigner(signer)
				header.SetHmacKeyVersion(uint32(1))
				hm := hmac.New(md5.New, []byte(key))
				hm.Write([]byte("some bytes"))
				header.SetHmac(hm.Sum(nil))
				hbytes, _ := proto.Marshal(header)

				framed := encodeMessage(hbytes, mbytes)
				unframed := splitter.UnframeRecord(framed, pack)
				c.Expect(pack.Signer, gs.Equals, "")
				// The function returns nil, and `unframed == nil` evaluates
				// to true, but `gs.IsNil` doesn't work here.
				c.Expect(string(unframed), gs.Equals, "")
			})
		})
	})
}
Exemple #27
0
func ReportSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	chanSize := pConfig.Globals.PluginChanSize

	checkForFields := func(c gs.Context, msg *message.Message) {
		f0Val, ok := msg.GetFieldValue(f0.GetName())
		c.Expect(ok, gs.IsTrue)
		c.Expect(f0Val.(int64), gs.Equals, f0.GetValue().(int64))
		f1Val, ok := msg.GetFieldValue(f1.GetName())
		c.Expect(ok, gs.IsTrue)
		c.Expect(f1Val.(string), gs.Equals, f1.GetValue().(string))
	}

	hasChannelData := func(msg *message.Message) (ok bool) {
		capVal, _ := msg.GetFieldValue("InChanCapacity")
		lenVal, _ := msg.GetFieldValue("InChanLength")
		var i int64
		if i, ok = capVal.(int64); !ok {
			return
		}
		if ok = (i == int64(chanSize)); !ok {
			return
		}
		if i, ok = lenVal.(int64); !ok {
			return
		}
		ok = (i == int64(0))
		return
	}

	fName := "counter"
	filter := new(CounterFilter)
	fRunner := NewFORunner(fName, filter, nil, chanSize)
	var err error
	fRunner.matcher, err = NewMatchRunner("Type == ''", "", fRunner, chanSize)
	c.Assume(err, gs.IsNil)
	fRunner.matcher.inChan = make(chan *PipelinePack, chanSize)
	leakCount := 10
	fRunner.SetLeakCount(leakCount)

	iName := "stat_accum"
	input := new(StatAccumInput)
	iRunner := NewInputRunner(iName, input, nil, false)

	c.Specify("`PopulateReportMsg`", func() {
		msg := ts.GetTestMessage()

		c.Specify("w/ a filter", func() {
			err := PopulateReportMsg(fRunner, msg)
			c.Assume(err, gs.IsNil)

			c.Specify("invokes `ReportMsg` on the filter", func() {
				checkForFields(c, msg)
			})

			c.Specify("adds the channel data", func() {
				c.Expect(hasChannelData(msg), gs.IsTrue)
			})

			c.Specify("has its leak count set properly", func() {
				leakVal, ok := msg.GetFieldValue("LeakCount")
				c.Assume(ok, gs.IsTrue)
				i, ok := leakVal.(int64)
				c.Assume(ok, gs.IsTrue)
				c.Expect(int(i), gs.Equals, leakCount)
			})
		})

		c.Specify("w/ an input", func() {
			err := PopulateReportMsg(iRunner, msg)
			c.Assume(err, gs.IsNil)

			c.Specify("invokes `ReportMsg` on the input", func() {
				checkForFields(c, msg)
			})

			c.Specify("doesn't add any channel data", func() {
				capVal, ok := msg.GetFieldValue("InChanCapacity")
				c.Expect(capVal, gs.IsNil)
				c.Expect(ok, gs.IsFalse)
				lenVal, ok := msg.GetFieldValue("InChanLength")
				c.Expect(lenVal, gs.IsNil)
				c.Expect(ok, gs.IsFalse)
			})
		})
	})

	c.Specify("PipelineConfig", func() {
		pc := NewPipelineConfig(nil)
		// Initialize all of the PipelinePacks that we'll need
		pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)

		pc.FilterRunners = map[string]FilterRunner{fName: fRunner}
		pc.InputRunners = map[string]InputRunner{iName: iRunner}

		c.Specify("returns full set of accurate reports", func() {
			reportChan := make(chan *PipelinePack)
			go pc.reports(reportChan)

			reports := make(map[string]*PipelinePack)
			for r := range reportChan {
				iName, ok := r.Message.GetFieldValue("name")
				c.Expect(ok, gs.IsTrue)
				name, ok := iName.(string)
				c.Expect(ok, gs.IsTrue)
				c.Expect(name, gs.Not(gs.Equals), "MISSING")
				reports[name] = r
				pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)
			}
			fReport := reports[fName]
			c.Expect(fReport, gs.Not(gs.IsNil))
			checkForFields(c, fReport.Message)
			c.Expect(hasChannelData(fReport.Message), gs.IsTrue)

			iReport := reports[iName]
			c.Expect(iReport, gs.Not(gs.IsNil))
			checkForFields(c, iReport.Message)

			recycleReport := reports["inputRecycleChan"]
			c.Expect(recycleReport, gs.Not(gs.IsNil))
			capVal, ok := recycleReport.Message.GetFieldValue("InChanCapacity")
			c.Expect(ok, gs.IsTrue)
			c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))

			injectReport := reports["injectRecycleChan"]
			c.Expect(injectReport, gs.Not(gs.IsNil))
			capVal, ok = injectReport.Message.GetFieldValue("InChanCapacity")
			c.Expect(ok, gs.IsTrue)
			c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))

			routerReport := reports["Router"]
			c.Expect(routerReport, gs.Not(gs.IsNil))
			c.Expect(hasChannelData(routerReport.Message), gs.IsTrue)
		})
	})
}
Exemple #28
0
func TcpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests")
	defer func() {
		ctrl.Finish()
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.Equals, nil)
	}()
	pConfig := NewPipelineConfig(nil)

	c.Specify("TcpOutput", func() {
		origBaseDir := Globals().BaseDir
		Globals().BaseDir = tmpDir
		defer func() {
			Globals().BaseDir = origBaseDir
		}()

		tcpOutput := new(TcpOutput)
		tcpOutput.SetName("test")
		config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
		tcpOutput.Init(config)

		tickChan := make(chan time.Time)
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan)
		encoder := new(ProtobufEncoder)
		encoder.Init(nil)

		var wg sync.WaitGroup
		inChan := make(chan *PipelinePack, 1)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		outStr := "Write me out to the network"
		newpack := NewPipelinePack(nil)
		newpack.Message = msg
		newpack.Decoded = true
		newpack.Message.SetPayload(outStr)
		matchBytes, err := proto.Marshal(newpack.Message)
		c.Expect(err, gs.IsNil)

		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)

		c.Specify("doesn't use framing w/o ProtobufEncoder", func() {
			encoder := new(plugins.PayloadEncoder)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			wg.Add(1)
			go func() {
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()

			close(inChan)
			wg.Wait()
			// We should fail if SetUseFraming is called since we didn't
			// EXPECT it.
		})

		c.Specify("doesn't use framing if config says not to", func() {
			useFraming := false
			config.UseFraming = &useFraming
			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			wg.Add(1)
			go func() {
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()

			close(inChan)
			wg.Wait()
			// We should fail if SetUseFraming is called since we didn't
			// EXPECT it.
		})

		c.Specify("writes out to the network", func() {
			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				ch <- "ready"
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch // wait for server

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().SetUseFraming(true)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()

			pack.Message.SetPayload(outStr)
			wg.Add(1)
			go func() {
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()

			msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			inChan <- pack
			result = <-ch

			msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(1))
			c.Expect(result, gs.Equals, string(matchBytes))

			close(inChan)
			wg.Wait() // wait for output to finish shutting down
		})

		c.Specify("far end not initially listening", func() {
			oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			pack.Message.SetPayload(outStr)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().SetUseFraming(true)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()

			wg.Add(1)
			go func() {
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()
			msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			inChan <- pack

			for x := 0; x < 5 && msgcount == 0; x++ {
				msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
				time.Sleep(time.Duration(100) * time.Millisecond)
			}

			// After the message is queued start the collector. However, we
			// don't have a way guarantee a send attempt has already been made
			// and that we are actually exercising the retry code.
			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch
			c.Expect(result, gs.Equals, string(matchBytes))

			close(inChan)
			wg.Wait() // wait for output to finish shutting down
		})
	})
}
Exemple #29
0
func UdpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A UdpInput", func() {
		udpInput := UdpInput{}
		err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr,
			Decoder:    "ProtobufDecoder",
			ParserType: "message.proto"})
		c.Assume(err, gs.IsNil)
		realListener := (udpInput.listener).(*net.UDPConn)
		c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)

		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
		ith.MockInputRunner.EXPECT().Name().Return("UdpInput")
		encCall := ith.MockHelper.EXPECT().DecoderRunner("ProtobufDecoder", "UdpInput-ProtobufDecoder")
		encCall.Return(ith.Decoder, true)

		c.Specify("reads a message from the connection and passes it to the decoder", func() {
			hbytes, _ := proto.Marshal(header)
			go func() {
				udpInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
			c.Assume(err, gs.IsNil)
			buf := encodeMessage(hbytes, mbytes)
			_, err = conn.Write(buf)
			c.Assume(err, gs.IsNil)
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			udpInput.Stop()
			c.Expect(ith.Pack, gs.Equals, packRef)
			c.Expect(string(ith.Pack.MsgBytes), gs.Equals, string(mbytes))
			c.Expect(ith.Pack.Decoded, gs.IsFalse)
		})
	})

	c.Specify("A UdpInput Multiline input", func() {
		ith.AddrStr = "localhost:55566"
		ith.ResolvedAddrStr = "127.0.0.1:55566"
		udpInput := UdpInput{}
		err := udpInput.Init(&UdpInputConfig{Net: "udp", Address: ith.AddrStr,
			Decoder:    "test",
			ParserType: "token"})
		c.Assume(err, gs.IsNil)
		realListener := (udpInput.listener).(*net.UDPConn)
		c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)

		mockDecoderRunner := ith.Decoder.(*pipelinemock.MockDecoderRunner)
		mockDecoderRunner.EXPECT().InChan().Return(ith.DecodeChan).Times(2)
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(2)
		ith.MockInputRunner.EXPECT().Name().Return("UdpInput").AnyTimes()
		encCall := ith.MockHelper.EXPECT().DecoderRunner("test", "UdpInput-test")
		encCall.Return(ith.Decoder, true)

		c.Specify("reads two messages from a packet and passes them to the decoder", func() {
			go func() {
				udpInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
			conn, err := net.Dial("udp", ith.AddrStr) // a mock connection will not work here since the mock read cannot block
			c.Assume(err, gs.IsNil)
			_, err = conn.Write([]byte("message1\nmessage2\n"))
			c.Assume(err, gs.IsNil)
			ith.PackSupply <- ith.Pack
			packRef := <-ith.DecodeChan
			c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message1\n")
			ith.PackSupply <- ith.Pack
			packRef = <-ith.DecodeChan
			c.Expect(string(packRef.Message.GetPayload()), gs.Equals, "message2\n")
			udpInput.Stop()
		})
	})
}
Exemple #30
0
func AMQPPluginSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)

	// Our two user/conn waitgroups
	ug := new(sync.WaitGroup)
	cg := new(sync.WaitGroup)

	// Setup the mock channel
	mch := NewMockAMQPChannel(ctrl)

	// Setup the mock amqpHub with the mock chan return
	aqh := NewMockAMQPConnectionHub(ctrl)
	aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)

	errChan := make(chan error, 1)

	c.Specify("An amqp input", func() {
		// Setup all the mock calls for Init
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)
		mch.EXPECT().QueueDeclare("", false, true, false, false,
			gomock.Any()).Return(amqp.Queue{}, nil)
		mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
		mch.EXPECT().Qos(2, 0, false).Return(nil)

		ith := new(plugins_ts.InputTestHelper)
		ith.Msg = pipeline_ts.GetTestMessage()
		ith.Pack = NewPipelinePack(config.InputRecycleChan())

		// set up mock helper, decoder set, and packSupply channel
		ith.MockHelper = NewMockPluginHelper(ctrl)
		ith.MockInputRunner = NewMockInputRunner(ctrl)
		mockDRunner := NewMockDecoderRunner(ctrl)
		ith.PackSupply = make(chan *PipelinePack, 1)
		ith.DecodeChan = make(chan *PipelinePack)

		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)

		amqpInput := new(AMQPInput)
		amqpInput.amqpHub = aqh
		config := amqpInput.ConfigStruct().(*AMQPInputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"
		config.QueueTTL = 300000

		c.Specify("with a valid setup and no decoder", func() {
			err := amqpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpInput.ch, gs.Equals, mch)

			c.Specify("consumes a message", func() {

				// Create a channel to send data to the input
				// Drop a message on there and close the channel
				streamChan := make(chan amqp.Delivery, 1)
				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)
				streamChan <- amqp.Delivery{
					ContentType:  "text/plain",
					Body:         []byte("This is a message"),
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the injected packet
				ith.MockInputRunner.EXPECT().Inject(gomock.Any())

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
				c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
				c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
			})
		})

		c.Specify("with a valid setup using a decoder", func() {
			decoderName := "defaultDecoder"
			config.Decoder = decoderName
			err := amqpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpInput.ch, gs.Equals, mch)

			// Mock up our default decoder runner and decoder.
			ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
			decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
			decCall.Return(mockDRunner, true)
			mockDecoder := NewMockDecoder(ctrl)
			mockDRunner.EXPECT().Decoder().Return(mockDecoder)

			c.Specify("consumes a message", func() {
				packs := []*PipelinePack{ith.Pack}
				mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)

				// Create a channel to send data to the input
				// Drop a message on there and close the channel
				streamChan := make(chan amqp.Delivery, 1)
				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)
				streamChan <- amqp.Delivery{
					ContentType:  "text/plain",
					Body:         []byte("This is a message"),
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the injected packet
				ith.MockInputRunner.EXPECT().Inject(gomock.Any())

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
				c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
			})

			c.Specify("consumes a serialized message", func() {
				encoder := client.NewProtobufEncoder(nil)
				streamChan := make(chan amqp.Delivery, 1)

				msg := new(message.Message)
				msg.SetUuid(uuid.NewRandom())
				msg.SetTimestamp(time.Now().UnixNano())
				msg.SetType("logfile")
				msg.SetLogger("/a/nice/path")
				msg.SetSeverity(int32(0))
				msg.SetEnvVersion("0.2")
				msg.SetPid(0)
				msg.SetPayload("This is a message")
				msg.SetHostname("TestHost")

				msgBody := make([]byte, 0, 500)
				_ = encoder.EncodeMessageStream(msg, &msgBody)

				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)

				streamChan <- amqp.Delivery{
					ContentType:  "application/hekad",
					Body:         msgBody,
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the decoded pack
				mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				packRef := <-ith.DecodeChan
				c.Expect(ith.Pack, gs.Equals, packRef)
				// Ignore leading 5 bytes of encoded message as thats the header
				c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})
	})

	c.Specify("An amqp output", func() {
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		pConfig := NewPipelineConfig(nil)

		amqpOutput := new(AMQPOutput)
		amqpOutput.amqpHub = aqh
		config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"

		closeChan := make(chan *amqp.Error)

		inChan := make(chan *PipelinePack, 1)

		mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)

		// Increase the usage since Run decrements it on close
		ug.Add(1)

		// Expect the close and the InChan calls
		aqh.EXPECT().Close("", cg)
		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("publishes a plain message", func() {
			encoder := new(plugins.PayloadEncoder)
			econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
			econfig.AppendNewlines = false
			encoder.Init(econfig)
			payloadBytes, err := encoder.Encode(pack)

			config.Encoder = "PayloadEncoder"
			config.ContentType = "text/plain"
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()

			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})

		c.Specify("publishes a serialized message", func() {
			encoder := new(ProtobufEncoder)
			encoder.SetPipelineConfig(pConfig)
			encoder.Init(nil)
			protoBytes, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()
			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})
	})
}