Пример #1
0
func DashboardOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := pipeline.NewPipelineConfig(nil)

	dashboardOutput := new(DashboardOutput)
	dashboardOutput.pConfig = pConfig

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)

	errChan := make(chan error, 1)

	startOutput := func() {
		go func() {
			errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
		}()
	}

	if runtime.GOOS != "windows" {
		c.Specify("A DashboardOutput", func() {

			tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
			c.Assume(err, gs.IsNil)
			config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
			config.WorkingDirectory = tmpdir

			c.Specify("Init halts if basedirectory is not writable", func() {
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)
				err = dashboardOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
			})

			c.Specify("that is running", func() {
				startedChan := make(chan bool, 1)
				defer close(startedChan)
				ts := httptest.NewUnstartedServer(nil)

				dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
					ts.Start()
					startedChan <- true
					return nil
				}

				ticker := make(chan time.Time)
				inChan := make(chan *pipeline.PipelinePack, 1)
				recycleChan := make(chan *pipeline.PipelinePack, 1)
				pack := pipeline.NewPipelinePack(recycleChan)
				pack.Message = pipeline_ts.GetTestMessage()

				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)

				err := os.MkdirAll(tmpdir, 0700)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)

				dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					// Noop
				})

				c.Specify("sets custom http headers", func() {
					config.Headers = http.Header{
						"One":  []string{"two", "three"},
						"Four": []string{"five", "six", "seven"},
					}
					err = dashboardOutput.Init(config)
					c.Assume(err, gs.IsNil)
					ts.Config = dashboardOutput.server

					startOutput()

					inChan <- pack
					<-startedChan
					resp, err := http.Get(ts.URL)
					c.Assume(err, gs.IsNil)
					resp.Body.Close()
					c.Assume(resp.StatusCode, gs.Equals, 200)

					// Verify headers are there
					eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
					c.Expect(eq, gs.IsTrue)
					eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
					c.Expect(eq, gs.IsTrue)
				})

				close(inChan)
				c.Expect(<-errChan, gs.IsNil)

				ts.Close()
			})
		})
	}
}
Пример #2
0
func UdpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	udpOutput := new(UdpOutput)
	config := udpOutput.ConfigStruct().(*UdpOutputConfig)

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	encoder := new(plugins.PayloadEncoder)
	encoder.Init(new(plugins.PayloadEncoderConfig))

	inChan := make(chan *pipeline.PipelinePack, 1)
	rChan := make(chan *pipeline.PipelinePack, 1)
	var wg sync.WaitGroup
	var rAddr net.Addr

	c.Specify("A UdpOutput", func() {
		msg := pipeline_ts.GetTestMessage()
		payload := "Write me out to the network."
		msg.SetPayload(payload)
		pack := pipeline.NewPipelinePack(rChan)
		pack.Message = msg

		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
		oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
		oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))

		c.Specify("using UDP", func() {
			addr := "127.0.0.1:45678"
			config.Address = addr
			ch := make(chan string, 1)

			collectData := func() {
				conn, err := net.ListenPacket("udp", addr)
				if err != nil {
					ch <- err.Error()
					return
				}

				ch <- "ready"
				b := make([]byte, 1000)
				var n int
				n, rAddr, _ = conn.ReadFrom(b)
				ch <- string(b[:n])
				conn.Close()
			}

			go collectData()
			result := <-ch // Wait for server to be ready.
			c.Assume(result, gs.Equals, "ready")

			c.Specify("writes out to the network", func() {
				err := udpOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				close(inChan)
				wg.Wait()
			})

			c.Specify("uses the specified local address", func() {
				config.LocalAddress = "localhost:12345"
				err := udpOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				c.Expect(rAddr.Network(), gs.Equals, "udp")
				c.Expect(rAddr.String(), gs.Equals, "127.0.0.1:12345")
				close(inChan)
				wg.Wait()
			})
		})

		c.Specify("using Unix datagrams", func() {
			if runtime.GOOS == "windows" {
				return
			}

			testUnixAddr := func() string {
				f, err := ioutil.TempFile("", "_heka_test_sock")
				c.Assume(err, gs.IsNil)
				addr := f.Name()
				f.Close()
				os.Remove(addr)
				return addr
			}

			config.Address = testUnixAddr()
			config.Net = "unixgram"
			ch := make(chan string, 1)
			var wg sync.WaitGroup
			var rAddr net.Addr

			collectData := func() {
				conn, err := net.ListenPacket("unixgram", config.Address)
				if err != nil {
					ch <- err.Error()
					return
				}

				ch <- "ready"
				b := make([]byte, 1000)
				var n int
				n, rAddr, _ = conn.ReadFrom(b)
				ch <- string(b[:n])
				conn.Close()
				err = os.Remove(config.Address)
				var errMsg string
				if err != nil {
					errMsg = err.Error()
				}
				ch <- errMsg
			}

			go collectData()
			result := <-ch // Wait for server to be ready.
			c.Assume(result, gs.Equals, "ready")

			c.Specify("writes out to the network", func() {
				err := udpOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				close(inChan)
				wg.Wait()
				result = <-ch
				c.Expect(result, gs.Equals, "")
			})

			c.Specify("uses the specified local address", func() {
				config.LocalAddress = testUnixAddr()
				err := udpOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				c.Expect(rAddr.Network(), gs.Equals, "unixgram")
				c.Expect(rAddr.String(), gs.Equals, config.LocalAddress)
				close(inChan)
				wg.Wait()
			})

		})
	})
	c.Specify("drop message contents if their size is bigger than allowed UDP datagram size", func() {
		huge_msg := pipeline_ts.GetTestMessage()
		payload := strings.Repeat("2", 131014)

		huge_msg.SetPayload(payload)

		huge_pack := pipeline.NewPipelinePack(rChan)
		huge_pack.Message = huge_msg

		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
		oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
		oth.MockOutputRunner.EXPECT().Encode(huge_pack).Return(encoder.Encode(huge_pack))
		oth.MockOutputRunner.EXPECT().LogError(fmt.Errorf("Message has exceeded allowed UDP data size: 131014 > 65507"))

		config.Address = "localhost:12345"
		err := udpOutput.Init(config)

		c.Assume(err, gs.IsNil)
		wg.Add(1)
		go func() {
			err = udpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.IsNil)
			wg.Done()
		}()
		inChan <- huge_pack

		close(inChan)
		wg.Wait()
	})

	c.Specify("checks validation of of Maximum message size limit", func() {

		config.Address = "localhost:12345"
		config.MaxMessageSize = 100

		err := udpOutput.Init(config)

		c.Assume(err.Error(), gs.Equals, "Maximum message size can't be smaller than 512 bytes.")
	})
}
Пример #3
0
func SmtpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	encoder := new(plugins.PayloadEncoder)
	econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
	econfig.AppendNewlines = false
	encoder.Init(econfig)

	c.Specify("A SmtpOutput", func() {
		smtpOutput := new(SmtpOutput)

		config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
		config.SendTo = []string{"root"}

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true
		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)
		runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
		runnerName.Return("SmtpOutput")
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()

		c.Specify("send email payload message", func() {
			err := smtpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			smtpOutput.sendFunction = testSendMail

			outStr := "Write me out to the network"
			pack.Message.SetPayload(outStr)
			go func() {
				wg.Add(1)
				smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
			inChan <- pack
			close(inChan)
			wg.Wait()
		})
	})

	// Use this test with a real server
	//  c.Specify("Real SmtpOutput output", func() {
	//  	smtpOutput := new(SmtpOutput)
	//
	//  	config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
	//  	config.SendTo = []string{"root"}
	//
	//  	msg := pipeline_ts.GetTestMessage()
	//  	pack := NewPipelinePack(pConfig.InputRecycleChan())
	//  	pack.Message = msg
	//  	pack.Decoded = true
	//  	inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
	//  	inChanCall.Return(inChan)
	//  	runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
	//  	runnerName.Return("SmtpOutput")
	//      oth.MockOutputRunner.EXPECT().Encoder().Return(encoder).AnyTimes()
	//
	//  	c.Specify("send a real email essage", func() {
	//
	//  		err := smtpOutput.Init(config)
	//  		c.Assume(err, gs.IsNil)
	//
	//  		outStr := "Write me out to the network"
	//  		pack.Message.SetPayload(outStr)
	//  		go func() {
	//  			wg.Add(1)
	//  			smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
	//  			wg.Done()
	//  		}()
	//  		inChan <- pack
	//  		time.Sleep(1000) // allow time for the message output
	//  		close(inChan)
	//  		wg.Wait()
	//  		// manually check the mail
	//  	})
	//  })
}
Пример #4
0
func TcpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests")
	defer func() {
		ctrl.Finish()
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.Equals, nil)
	}()
	pConfig := NewPipelineConfig(nil)

	c.Specify("TcpOutput", func() {
		origBaseDir := Globals().BaseDir
		Globals().BaseDir = tmpDir
		defer func() {
			Globals().BaseDir = origBaseDir
		}()

		tcpOutput := new(TcpOutput)
		tcpOutput.SetName("test")
		config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
		tcpOutput.Init(config)

		tickChan := make(chan time.Time)
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan)
		encoder := new(ProtobufEncoder)
		encoder.Init(nil)

		var wg sync.WaitGroup
		inChan := make(chan *PipelinePack, 1)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		outStr := "Write me out to the network"
		newpack := NewPipelinePack(nil)
		newpack.Message = msg
		newpack.Decoded = true
		newpack.Message.SetPayload(outStr)
		matchBytes, err := proto.Marshal(newpack.Message)
		c.Expect(err, gs.IsNil)

		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)

		c.Specify("doesn't use framing w/o ProtobufEncoder", func() {
			encoder := new(plugins.PayloadEncoder)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			wg.Add(1)
			go func() {
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()

			close(inChan)
			wg.Wait()
			// We should fail if SetUseFraming is called since we didn't
			// EXPECT it.
		})

		c.Specify("doesn't use framing if config says not to", func() {
			useFraming := false
			config.UseFraming = &useFraming
			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			wg.Add(1)
			go func() {
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()

			close(inChan)
			wg.Wait()
			// We should fail if SetUseFraming is called since we didn't
			// EXPECT it.
		})

		c.Specify("writes out to the network", func() {
			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				ch <- "ready"
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch // wait for server

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().SetUseFraming(true)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()

			pack.Message.SetPayload(outStr)
			wg.Add(1)
			go func() {
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()

			msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			inChan <- pack
			result = <-ch

			msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(1))
			c.Expect(result, gs.Equals, string(matchBytes))

			close(inChan)
			wg.Wait() // wait for output to finish shutting down
		})

		c.Specify("far end not initially listening", func() {
			oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			pack.Message.SetPayload(outStr)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().SetUseFraming(true)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()

			wg.Add(1)
			go func() {
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()
			msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			inChan <- pack

			for x := 0; x < 5 && msgcount == 0; x++ {
				msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
				time.Sleep(time.Duration(100) * time.Millisecond)
			}

			// After the message is queued start the collector. However, we
			// don't have a way guarantee a send attempt has already been made
			// and that we are actually exercising the retry code.
			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch
			c.Expect(result, gs.Equals, string(matchBytes))

			close(inChan)
			wg.Wait() // wait for output to finish shutting down
		})
	})
}
Пример #5
0
func AMQPPluginSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)

	// Our two user/conn waitgroups
	ug := new(sync.WaitGroup)
	cg := new(sync.WaitGroup)

	// Setup the mock channel
	mch := NewMockAMQPChannel(ctrl)

	// Setup the mock amqpHub with the mock chan return
	aqh := NewMockAMQPConnectionHub(ctrl)
	aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)

	errChan := make(chan error, 1)

	c.Specify("An amqp input", func() {
		// Setup all the mock calls for Init
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)
		mch.EXPECT().QueueDeclare("", false, true, false, false,
			gomock.Any()).Return(amqp.Queue{}, nil)
		mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
		mch.EXPECT().Qos(2, 0, false).Return(nil)

		ith := new(plugins_ts.InputTestHelper)
		ith.Msg = pipeline_ts.GetTestMessage()
		ith.Pack = NewPipelinePack(config.InputRecycleChan())

		// set up mock helper, decoder set, and packSupply channel
		ith.MockHelper = NewMockPluginHelper(ctrl)
		ith.MockInputRunner = NewMockInputRunner(ctrl)
		mockDRunner := NewMockDecoderRunner(ctrl)
		ith.PackSupply = make(chan *PipelinePack, 1)
		ith.DecodeChan = make(chan *PipelinePack)

		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)

		amqpInput := new(AMQPInput)
		amqpInput.amqpHub = aqh
		config := amqpInput.ConfigStruct().(*AMQPInputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"
		config.QueueTTL = 300000

		c.Specify("with a valid setup and no decoder", func() {
			err := amqpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpInput.ch, gs.Equals, mch)

			c.Specify("consumes a message", func() {

				// Create a channel to send data to the input
				// Drop a message on there and close the channel
				streamChan := make(chan amqp.Delivery, 1)
				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)
				streamChan <- amqp.Delivery{
					ContentType:  "text/plain",
					Body:         []byte("This is a message"),
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the injected packet
				ith.MockInputRunner.EXPECT().Inject(gomock.Any())

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
				c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
				c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
			})
		})

		c.Specify("with a valid setup using a decoder", func() {
			decoderName := "defaultDecoder"
			config.Decoder = decoderName
			err := amqpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpInput.ch, gs.Equals, mch)

			// Mock up our default decoder runner and decoder.
			ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
			decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
			decCall.Return(mockDRunner, true)
			mockDecoder := NewMockDecoder(ctrl)
			mockDRunner.EXPECT().Decoder().Return(mockDecoder)

			c.Specify("consumes a message", func() {
				packs := []*PipelinePack{ith.Pack}
				mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)

				// Create a channel to send data to the input
				// Drop a message on there and close the channel
				streamChan := make(chan amqp.Delivery, 1)
				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)
				streamChan <- amqp.Delivery{
					ContentType:  "text/plain",
					Body:         []byte("This is a message"),
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the injected packet
				ith.MockInputRunner.EXPECT().Inject(gomock.Any())

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
				c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
			})

			c.Specify("consumes a serialized message", func() {
				encoder := client.NewProtobufEncoder(nil)
				streamChan := make(chan amqp.Delivery, 1)

				msg := new(message.Message)
				msg.SetUuid(uuid.NewRandom())
				msg.SetTimestamp(time.Now().UnixNano())
				msg.SetType("logfile")
				msg.SetLogger("/a/nice/path")
				msg.SetSeverity(int32(0))
				msg.SetEnvVersion("0.2")
				msg.SetPid(0)
				msg.SetPayload("This is a message")
				msg.SetHostname("TestHost")

				msgBody := make([]byte, 0, 500)
				_ = encoder.EncodeMessageStream(msg, &msgBody)

				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)

				streamChan <- amqp.Delivery{
					ContentType:  "application/hekad",
					Body:         msgBody,
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the decoded pack
				mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				packRef := <-ith.DecodeChan
				c.Expect(ith.Pack, gs.Equals, packRef)
				// Ignore leading 5 bytes of encoded message as thats the header
				c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})
	})

	c.Specify("An amqp output", func() {
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		pConfig := NewPipelineConfig(nil)

		amqpOutput := new(AMQPOutput)
		amqpOutput.amqpHub = aqh
		config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"

		closeChan := make(chan *amqp.Error)

		inChan := make(chan *PipelinePack, 1)

		mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)

		// Increase the usage since Run decrements it on close
		ug.Add(1)

		// Expect the close and the InChan calls
		aqh.EXPECT().Close("", cg)
		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("publishes a plain message", func() {
			encoder := new(plugins.PayloadEncoder)
			econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
			econfig.AppendNewlines = false
			encoder.Init(econfig)
			payloadBytes, err := encoder.Encode(pack)

			config.Encoder = "PayloadEncoder"
			config.ContentType = "text/plain"
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()

			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})

		c.Specify("publishes a serialized message", func() {
			encoder := new(ProtobufEncoder)
			encoder.SetPipelineConfig(pConfig)
			encoder.Init(nil)
			protoBytes, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()
			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})
	})
}
Пример #6
0
func IrcOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pipelineConfig := NewPipelineConfig(nil)
	outTestHelper := plugins_ts.NewOutputTestHelper(ctrl)

	var wg sync.WaitGroup

	errChan := make(chan error, 1)
	tickChan := make(chan time.Time)
	inChan := make(chan *PipelinePack, 5)

	c.Specify("An IrcOutput", func() {
		ircOutput := new(IrcOutput)
		ircOutput.InitIrcCon = NewMockIrcConn
		encoder := new(plugins.PayloadEncoder)
		encoder.Init(&plugins.PayloadEncoderConfig{})
		config := ircOutput.ConfigStruct().(*IrcOutputConfig)
		config.Server = "irc.example.org"
		config.Nick = "heka_bot"
		config.Ident = "heka"
		config.Channels = []string{"#test_channel"}

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pipelineConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("requires an encoder", func() {
			err := ircOutput.Init(config)
			c.Assume(err, gs.IsNil)

			outTestHelper.MockOutputRunner.EXPECT().Encoder().Return(nil)
			err = ircOutput.Run(outTestHelper.MockOutputRunner, outTestHelper.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))

		})

		c.Specify("that is started", func() {

			outTestHelper.MockOutputRunner.EXPECT().Ticker().Return(tickChan)
			outTestHelper.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			outTestHelper.MockOutputRunner.EXPECT().InChan().Return(inChan)
			outTestHelper.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes()

			startOutput := func() {
				wg.Add(1)
				go func() {
					err := ircOutput.Run(outTestHelper.MockOutputRunner, outTestHelper.MockHelper)
					errChan <- err
					wg.Done()
				}()
				<-mockIrcConn.connected
			}

			c.Specify("sends incoming messages to an irc server", func() {
				err := ircOutput.Init(config)
				c.Assume(err, gs.IsNil)

				c.Assume(len(ircOutput.Channels), gs.Equals, 1)
				ircChan := ircOutput.Channels[0]

				startOutput()

				// Send the data
				inChan <- pack
				// wait for it to arrive
				p := <-ircOutput.OutQueue
				ircOutput.OutQueue <- p
				// once it's arrived send a tick so it gets processed
				tickChan <- time.Now()

				close(inChan)
				wg.Wait()

				// Verify we've exited
				c.Expect(mockIrcConn.quit, gs.IsTrue)

				// verify the message was sent
				msgs := mockIrcConn.msgs[ircChan]
				c.Expect(len(msgs), gs.Equals, 1)
				c.Expect(msgs[0], gs.Equals, string(*msg.Payload))
			})

			c.Specify("drops messages when outqueue is full", func() {
				config.QueueSize = 1
				err := ircOutput.Init(config)
				c.Assume(err, gs.IsNil)

				c.Assume(len(ircOutput.Channels), gs.Equals, 1)
				ircChan := ircOutput.Channels[0]

				outTestHelper.MockOutputRunner.EXPECT().LogError(
					ErrOutQueueFull)
				outTestHelper.MockOutputRunner.EXPECT().LogError(
					fmt.Errorf("%s Channel: %s.", ErrBacklogQueueFull, ircChan))

				startOutput()

				// Need to wait for callbacks to get registered before calling part.
				// If we've called connected, then callbacks have been registered.

				// Leave the irc channel so we can fill up the backlog
				ircOutput.Conn.Part(ircChan)
				c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)

				// Put some data into the inChan
				inChan <- pack

				// Wait for it to arrive in the OutQueue and put it back
				// We *must* do this before sending the tick to avoid data races
				p := <-ircOutput.OutQueue
				ircOutput.OutQueue <- p

				// One tick so that it goes from the OutQueue into the BacklogQueue
				tickChan <- time.Now()

				// Verify the item is in the backlog queue and put it back
				// (It will never leave this loop if the msg never arrives)
				queue := ircOutput.BacklogQueues[0]
				msg := <-queue
				queue <- msg
				c.Expect(len(queue), gs.Equals, 1)

				// Send another message to fill the OutQueue
				inChan <- pack

				// Wait for it to arrive, again
				p = <-ircOutput.OutQueue
				ircOutput.OutQueue <- p

				// Send the tick after it's in the OutQueue so we can try processing
				// it. This is where we drop it since we cant send, and the
				// BacklogQueue is already full.
				tickChan <- time.Now()

				// Now we want to also cause the OutQueue to drop a message.
				// We don't have to wait for it to arrive in the OutQueue like we do above
				// since it shouldn't make it there.
				inChan <- pack
				inChan <- pack

				close(inChan)
				wg.Wait()

				// Verify we've exited
				c.Expect(mockIrcConn.quit, gs.IsTrue)

				// verify the backlog queue is empty
				c.Expect(len(queue), gs.Equals, 0)

				// verify that no messages were sent.
				msgs := mockIrcConn.msgs[ircChan]
				c.Expect(len(msgs), gs.Equals, 0)
			})

			c.Specify("automatically reconnects on disconnect", func() {
				config.TimeBeforeReconnect = 0
				err := ircOutput.Init(config)
				c.Assume(err, gs.IsNil)

				outTestHelper.MockOutputRunner.EXPECT().LogMessage(DisconnectMsg)
				outTestHelper.MockOutputRunner.EXPECT().LogMessage(ReconnectedMsg)

				startOutput()

				ircOutput.Conn.Disconnect()
				// verify we've been disconnected
				c.Expect(<-mockIrcConn.connected, gs.IsFalse)

				// Wait to become reconnected.
				c.Expect(<-mockIrcConn.connected, gs.IsTrue)

				close(inChan)
				wg.Wait()

				c.Expect(mockIrcConn.quit, gs.IsTrue)
			})

			c.Specify("logs an error and exits from Run() when it cannot reconnect", func() {
				config.TimeBeforeReconnect = 0

				err := ircOutput.Init(config)
				c.Assume(err, gs.IsNil)

				// Explicitly fail to reconnect
				mockIrcConn.failReconnect = true

				outTestHelper.MockOutputRunner.EXPECT().LogMessage(DisconnectMsg)
				outTestHelper.MockOutputRunner.EXPECT().LogError(fmt.Errorf(ErrReconnecting, mockErrFailReconnect))

				startOutput()

				// Need to wait for callbacks to get registered before we disconnect

				ircOutput.Conn.Disconnect()
				// verify we've been disconnected

				// Run() should return without closing the inChan because the plugin
				// automatically cleans up and returns if it fails to reconnect
				wg.Wait()

				c.Expect(mockIrcConn.quit, gs.IsTrue)
				// do this at the end for test cleanup purposes
				close(inChan)
			})

			c.Specify("when kicked from an irc channel", func() {

				c.Specify("rejoins when configured to do so", func() {
					config.RejoinOnKick = true
					err := ircOutput.Init(config)
					c.Assume(err, gs.IsNil)

					c.Assume(len(ircOutput.Channels), gs.Equals, 1)
					ircChan := ircOutput.Channels[0]

					startOutput()

					// trigger a kick event for the irc channel
					kick(mockIrcConn, ircChan)
					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)
					// quit
					close(inChan)
				})

				c.Specify("doesnt rejoin when it isnt configured to", func() {
					config.RejoinOnKick = false // this is already the default
					err := ircOutput.Init(config)
					c.Assume(err, gs.IsNil)

					c.Assume(len(ircOutput.Channels), gs.Equals, 1)
					ircChan := ircOutput.Channels[0]

					startOutput()

					// Since this is the only channel we're in, we should get an
					// error that there are no channels to join left after being
					// kicked
					outTestHelper.MockOutputRunner.EXPECT().LogError(ErrNoJoinableChannels)
					// trigger a kick for the irc channel
					kick(mockIrcConn, ircChan)
					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
					// We shouldnt need to close the inChan, since we have no
					// joinable channels, we should be cleaning up already.
				})

				wg.Wait()
			})

			c.Specify("when trying to join an unjoinable channel", func() {
				c.Specify("it logs an error", func() {
					config.Channels = []string{"foo", "baz"}
					err := ircOutput.Init(config)
					c.Assume(err, gs.IsNil)

					c.Assume(len(ircOutput.Channels), gs.Equals, 2)
					ircChan := ircOutput.Channels[0]

					startOutput()

					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)

					reason := "foo"

					err = IrcCannotJoinError{ircChan, reason}
					outTestHelper.MockOutputRunner.EXPECT().LogError(err)

					args := []string{"", ircChan, reason}
					event := irc.Event{Arguments: args}

					c.Specify("when the channel key is wrong", func() {
						event.Code = IRC_ERR_BADCHANNELKEY
					})

					c.Specify("when banned from the channel", func() {
						event.Code = IRC_ERR_BANNEDFROMCHAN
					})

					c.Specify("when there is no such channel", func() {
						event.Code = IRC_NOSUCHCHANNEL
					})

					c.Specify("when the channel is invite only", func() {
						event.Code = IRC_ERR_INVITEONLYCHAN
					})

					ircOutput.Conn.RunCallbacks(&event)

					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)

					// should still be unable to join
					ircOutput.Join(ircChan)
					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)

					close(inChan)
				})

				c.Specify("it exits if there are no joinable channels", func() {
					err := ircOutput.Init(config)
					c.Assume(err, gs.IsNil)

					c.Assume(len(ircOutput.Channels), gs.Equals, 1)
					ircChan := ircOutput.Channels[0]

					startOutput()

					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)

					reason := "foo"

					err = IrcCannotJoinError{ircChan, reason}
					outTestHelper.MockOutputRunner.EXPECT().LogError(err)
					outTestHelper.MockOutputRunner.EXPECT().LogError(ErrNoJoinableChannels)

					args := []string{"", ircChan, reason}
					event := irc.Event{Arguments: args, Code: IRC_ERR_BADCHANNELKEY}

					ircOutput.Conn.RunCallbacks(&event)

					// No close since we should exit without.
				})

				wg.Wait()
			})

			c.Specify("when trying to join a full channel", func() {
				config.TimeBeforeRejoin = 0
				err := ircOutput.Init(config)
				c.Assume(err, gs.IsNil)

				c.Assume(len(ircOutput.Channels), gs.Equals, 1)
				ircChan := ircOutput.Channels[0]

				startOutput()

				c.Expect(ircOutput.JoinedChannels[0], gs.Equals, JOINED)

				// We should be able to join before this
				ircOutput.Join(ircChan)

				reason := "full channel"
				err = fmt.Errorf("%s. Retrying in %d seconds.",
					IrcCannotJoinError{ircChan, reason}.Error(),
					ircOutput.TimeBeforeRejoin)

				args := []string{"", ircChan, reason}
				event := irc.Event{Code: IRC_ERR_CHANNELISFULL, Arguments: args}

				c.Specify("logs an error after using up its attempts", func() {
					mockIrcConn.failJoin = true
					// We should leave the channel so that we aren't just
					// causing a failure to join even when we're already in the
					// channel
					ircOutput.Conn.Part(ircChan)

					maxRetries := int(ircOutput.MaxJoinRetries)
					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)

					gomock.InOrder(
						outTestHelper.MockOutputRunner.EXPECT().LogError(
							err).Times(maxRetries),
						outTestHelper.MockOutputRunner.EXPECT().LogError(
							fmt.Errorf(ErrUsedUpRetryAttempts, ircChan)),
						outTestHelper.MockOutputRunner.EXPECT().LogError(
							ErrNoJoinableChannels),
					)

					for i := 1; i <= maxRetries; i++ {
						// Trigger the event the number of times
						ircOutput.Conn.RunCallbacks(&event)
						c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, i)

					}
					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, NOTJOINED)
					ircOutput.Conn.RunCallbacks(&event)
					// We've used up our attempts,
					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
					ircOutput.Conn.RunCallbacks(&event)
					c.Expect(ircOutput.JoinedChannels[0], gs.Equals, CANNOTJOIN)
				})

				c.Specify("resets its attempts when it successful", func() {
					mockIrcConn.failJoin = true
					outTestHelper.MockOutputRunner.EXPECT().LogError(err).Times(2)
					ircOutput.Conn.RunCallbacks(&event)
					c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, 1)
					mockIrcConn.failJoin = false
					ircOutput.Conn.RunCallbacks(&event)
					c.Expect(int(ircOutput.numRetries[ircChan]), gs.Equals, 0)
				})

				close(inChan)
				wg.Wait()
			})

			c.Specify("when banned from a server logs an error and quits", func() {
				err := ircOutput.Init(config)
				c.Assume(err, gs.IsNil)

				startOutput()

				event := irc.Event{Code: IRC_ERR_YOUREBANNEDCREEP}
				outTestHelper.MockOutputRunner.EXPECT().LogError(ErrBannedFromServer)
				ircOutput.Conn.RunCallbacks(&event)

				// We should be able to exit without closing the channel first
				wg.Wait()

				// Clean this up
				close(inChan)
			})

			c.Expect(<-errChan, gs.IsNil)
			c.Expect(<-mockIrcConn.connected, gs.IsFalse)

		})

		// Cleanup which should happen each run
		close(mockIrcConn.connected)
		close(mockIrcConn.delivered)
		close(mockIrcConn.Error)
	})

}
Пример #7
0
func FileOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
	tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)

	defer func() {
		ctrl.Finish()
		os.Remove(tmpFilePath)
	}()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	c.Specify("A FileOutput", func() {
		fileOutput := new(FileOutput)
		encoder := new(plugins.PayloadEncoder)
		encoder.Init(encoder.ConfigStruct())

		config := fileOutput.ConfigStruct().(*FileOutputConfig)
		config.Path = tmpFilePath

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("w/ ProtobufEncoder", func() {
			encoder := new(ProtobufEncoder)
			encoder.Init(nil)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)

			c.Specify("uses framing", func() {
				oth.MockOutputRunner.EXPECT().SetUseFraming(true)
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)
				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				wg.Add(1)
				go func() {
					err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()
				close(inChan)
				wg.Wait()
			})

			c.Specify("but not if config says not to", func() {
				useFraming := false
				config.UseFraming = &useFraming
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)
				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				wg.Add(1)
				go func() {
					err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()
				close(inChan)
				wg.Wait()
				// We should fail if SetUseFraming is called since we didn't
				// EXPECT it.

			})
		})

		c.Specify("processes incoming messages", func() {
			err := fileOutput.Init(config)
			c.Assume(err, gs.IsNil)
			fileOutput.file.Close()
			// Save for comparison.
			payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())

			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			wg.Add(1)
			go fileOutput.receiver(oth.MockOutputRunner, &wg)
			inChan <- pack
			close(inChan)
			outBatch := <-fileOutput.batchChan
			wg.Wait()
			c.Expect(string(outBatch), gs.Equals, payload)
		})

		c.Specify("commits to a file", func() {
			outStr := "Write me out to the log file"
			outBytes := []byte(outStr)

			c.Specify("with default settings", func() {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				contents, err := ioutil.ReadAll(tmpFile)
				c.Assume(err, gs.IsNil)
				c.Expect(string(contents), gs.Equals, outStr)
			})

			c.Specify("with different Perm settings", func() {
				config.Perm = "600"
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				fileInfo, err := tmpFile.Stat()
				c.Assume(err, gs.IsNil)
				fileMode := fileInfo.Mode()
				if runtime.GOOS == "windows" {
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
				} else {
					// 7 consecutive dashes implies no perms for group or other
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
				}
			})
		})

		if runtime.GOOS != "windows" {
			c.Specify("Init halts if basedirectory is not writable", func() {
				tmpdir := filepath.Join(os.TempDir(), "tmpdir")
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				config.Path = filepath.Join(tmpdir, "out.txt")
				err = fileOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
				os.RemoveAll(tmpdir)
			})

			c.Specify("honors folder_perm setting", func() {
				config.FolderPerm = "750"
				subdir := filepath.Join(os.TempDir(), "subdir")
				config.Path = filepath.Join(subdir, "out.txt")
				err := fileOutput.Init(config)
				defer os.RemoveAll(subdir)
				c.Assume(err, gs.IsNil)

				fi, err := os.Stat(subdir)
				c.Expect(fi.IsDir(), gs.IsTrue)
				c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
			})
		}

		c.Specify("that starts receiving w/ a flush interval", func() {
			config.FlushInterval = 100000000 // We'll trigger the timer manually.
			inChan := make(chan *PipelinePack)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			timerChan := make(chan time.Time)

			msg2 := pipeline_ts.GetTestMessage()
			pack2 := NewPipelinePack(pConfig.InputRecycleChan())
			pack2.Message = msg2

			recvWithConfig := func(config *FileOutputConfig) {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)
				wg.Add(1)
				go fileOutput.receiver(oth.MockOutputRunner, &wg)
				runtime.Gosched() // Yield so we can overwrite the timerChan.
				fileOutput.timerChan = timerChan
			}

			cleanUp := func() {
				close(inChan)
				fileOutput.file.Close()
				wg.Done()
			}

			c.Specify("honors flush interval", func() {
				oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack
				select {
				case _ = <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}
				timerChan <- time.Now()
				select {
				case _ = <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval AND flush count", func() {
				oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
				oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
				config.FlushCount = 2
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				timerChan <- time.Now()
				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				inChan <- pack2
				runtime.Gosched()
				select {
				case <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval OR flush count", func() {
				oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
				config.FlushCount = 2
				config.FlushOperator = "OR"
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				c.Specify("when interval triggers first", func() {
					timerChan <- time.Now()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})

				c.Specify("when count triggers first", func() {
					out, err := encoder.Encode(pack2)
					oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
					inChan <- pack2
					runtime.Gosched()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})
			})
		})
	})
}
Пример #8
0
func AMQPPluginSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)

	// Our two user/conn waitgroups.
	ug := new(sync.WaitGroup)
	cg := new(sync.WaitGroup)

	// Setup the mock channel.
	mch := NewMockAMQPChannel(ctrl)

	// Setup the mock amqpHub with the mock chan return.
	aqh := NewMockAMQPConnectionHub(ctrl)
	aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)

	errChan := make(chan error, 1)
	bytesChan := make(chan []byte, 1)

	c.Specify("An amqp input", func() {
		// Setup all the mock calls for Init.
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)
		mch.EXPECT().QueueDeclare("", false, true, false, false,
			gomock.Any()).Return(amqp.Queue{}, nil)
		mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
		mch.EXPECT().Qos(2, 0, false).Return(nil)

		ith := new(plugins_ts.InputTestHelper)
		ith.Msg = pipeline_ts.GetTestMessage()
		ith.Pack = NewPipelinePack(config.InputRecycleChan())

		// Set up relevant mocks.
		ith.MockHelper = NewMockPluginHelper(ctrl)
		ith.MockInputRunner = NewMockInputRunner(ctrl)
		ith.MockSplitterRunner = NewMockSplitterRunner(ctrl)
		ith.PackSupply = make(chan *PipelinePack, 1)

		ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)

		amqpInput := new(AMQPInput)
		amqpInput.amqpHub = aqh
		config := amqpInput.ConfigStruct().(*AMQPInputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"
		config.QueueTTL = 300000

		err := amqpInput.Init(config)
		c.Assume(err, gs.IsNil)
		c.Expect(amqpInput.ch, gs.Equals, mch)

		c.Specify("consumes a text message", func() {
			// Create a channel to send data to the input. Drop a message on
			// there and close the channel.
			streamChan := make(chan amqp.Delivery, 1)
			ack := plugins_ts.NewMockAcknowledger(ctrl)
			ack.EXPECT().Ack(gomock.Any(), false)
			streamChan <- amqp.Delivery{
				ContentType:  "text/plain",
				Body:         []byte("This is a message"),
				Timestamp:    time.Now(),
				Acknowledger: ack,
			}
			mch.EXPECT().Consume("", "", false, false, false, false,
				gomock.Any()).Return(streamChan, nil)

			// Increase the usage since Run decrements it on close.
			ug.Add(1)

			splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
				nil)
			splitCall.Do(func(recd []byte, del Deliverer) {
				bytesChan <- recd
			})
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			go func() {
				err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
				errChan <- err
			}()

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "This is a message")
			close(streamChan)
			err = <-errChan
		})

		c.Specify("consumes a protobuf encoded message", func() {
			encoder := client.NewProtobufEncoder(nil)
			streamChan := make(chan amqp.Delivery, 1)

			msg := new(message.Message)
			msg.SetUuid(uuid.NewRandom())
			msg.SetTimestamp(time.Now().UnixNano())
			msg.SetType("logfile")
			msg.SetLogger("/a/nice/path")
			msg.SetSeverity(int32(0))
			msg.SetEnvVersion("0.2")
			msg.SetPid(0)
			msg.SetPayload("This is a message")
			msg.SetHostname("TestHost")

			msgBody := make([]byte, 0, 500)
			_ = encoder.EncodeMessageStream(msg, &msgBody)

			ack := plugins_ts.NewMockAcknowledger(ctrl)
			ack.EXPECT().Ack(gomock.Any(), false)

			streamChan <- amqp.Delivery{
				ContentType:  "application/hekad",
				Body:         msgBody,
				Timestamp:    time.Now(),
				Acknowledger: ack,
			}
			mch.EXPECT().Consume("", "", false, false, false, false,
				gomock.Any()).Return(streamChan, nil)

			// Increase the usage since Run decrements it on close.
			ug.Add(1)

			splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
				nil)
			splitCall.Do(func(recd []byte, del Deliverer) {
				bytesChan <- recd
			})
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
			go func() {
				err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
				errChan <- err
			}()

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, string(msgBody))
			close(streamChan)
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})
	})

	c.Specify("An amqp output", func() {
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		pConfig := NewPipelineConfig(nil)

		amqpOutput := new(AMQPOutput)
		amqpOutput.amqpHub = aqh
		config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"

		closeChan := make(chan *amqp.Error)

		inChan := make(chan *PipelinePack, 1)

		mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)

		// Increase the usage since Run decrements it on close.
		ug.Add(1)

		// Expect the close and the InChan calls.
		aqh.EXPECT().Close("", cg)
		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("publishes a plain message", func() {
			encoder := new(plugins.PayloadEncoder)
			econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
			econfig.AppendNewlines = false
			encoder.Init(econfig)
			payloadBytes, err := encoder.Encode(pack)

			config.Encoder = "PayloadEncoder"
			config.ContentType = "text/plain"
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()

			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})

		c.Specify("publishes a serialized message", func() {
			encoder := new(ProtobufEncoder)
			encoder.SetPipelineConfig(pConfig)
			encoder.Init(nil)
			protoBytes, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()
			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})
	})
}
Пример #9
0
func TcpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests-")
	os.MkdirAll(tmpDir, 0777)
	defer func() {
		ctrl.Finish()
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.Equals, nil)
	}()
	pConfig := NewPipelineConfig(nil)

	c.Specify("TcpOutput Internals", func() {
		tcpOutput := new(TcpOutput)
		tcpOutput.connection = pipeline_ts.NewMockConn(ctrl)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("SetName", func() {
			tcpOutput.SetName("this-is a test")
			c.Expect(tcpOutput.name, gs.Equals, "this_is_a_test")
		})

		c.Specify("fileExists", func() {
			c.Expect(fileExists(tmpDir), gs.IsTrue)
			c.Expect(fileExists(filepath.Join(tmpDir, "test.log")), gs.IsFalse)
		})

		c.Specify("extractBufferId", func() {
			id, err := extractBufferId("555.log")
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(555))
			id, err = extractBufferId("")
			c.Expect(err, gs.Not(gs.IsNil))
			id, err = extractBufferId("a.log")
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("findBufferId", func() {
			c.Expect(findBufferId(tmpDir, true), gs.Equals, uint(0))
			c.Expect(findBufferId(tmpDir, false), gs.Equals, uint(0))
			_, err := os.OpenFile(filepath.Join(tmpDir, "4.log"), os.O_CREATE, 0644)
			c.Expect(err, gs.IsNil)
			_, err = os.OpenFile(filepath.Join(tmpDir, "5.log"), os.O_CREATE, 0644)
			c.Expect(err, gs.IsNil)
			_, err = os.OpenFile(filepath.Join(tmpDir, "6a.log"), os.O_CREATE, 0644)
			c.Expect(err, gs.IsNil)
			c.Expect(findBufferId(tmpDir, false), gs.Equals, uint(4))
			c.Expect(findBufferId(tmpDir, true), gs.Equals, uint(5))
		})

		c.Specify("writeCheckpoint", func() {
			tcpOutput.checkpointFilename = filepath.Join(tmpDir, "cp.txt")
			err := tcpOutput.writeCheckpoint(43, 99999)
			c.Expect(err, gs.IsNil)
			c.Expect(fileExists(tcpOutput.checkpointFilename), gs.IsTrue)

			id, offset, err := readCheckpoint(tcpOutput.checkpointFilename)
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(43))
			c.Expect(offset, gs.Equals, int64(99999))

			err = tcpOutput.writeCheckpoint(43, 1)
			c.Expect(err, gs.IsNil)
			id, offset, err = readCheckpoint(tcpOutput.checkpointFilename)
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(43))
			c.Expect(offset, gs.Equals, int64(1))
			tcpOutput.checkpointFile.Close()
		})

		c.Specify("readCheckpoint", func() {
			cp := filepath.Join(tmpDir, "cp.txt")
			file, err := os.OpenFile(cp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
			c.Expect(err, gs.IsNil)
			id, offset, err := readCheckpoint(cp)
			c.Expect(err, gs.Not(gs.IsNil))

			file.WriteString("22")
			id, offset, err = readCheckpoint(cp)
			c.Expect(err.Error(), gs.Equals, "invalid checkpoint format")

			file.Seek(0, 0)
			file.WriteString("aa 22")
			id, offset, err = readCheckpoint(cp)
			c.Expect(err.Error(), gs.Equals, "invalid checkpoint id")

			file.Seek(0, 0)
			file.WriteString("43 aa")
			id, offset, err = readCheckpoint(cp)
			c.Expect(err.Error(), gs.Equals, "invalid checkpoint offset")

			file.Seek(0, 0)
			file.WriteString("43 22")
			file.Close()

			id, offset, err = readCheckpoint(cp)
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(43))
			c.Expect(offset, gs.Equals, int64(22))
		})

		c.Specify("writeToNextFile", func() {
			tcpOutput.checkpointFilename = filepath.Join(tmpDir, "cp.txt")
			tcpOutput.queue = tmpDir
			err := tcpOutput.writeToNextFile()
			c.Expect(err, gs.IsNil)
			c.Expect(fileExists(getQueueFilename(tcpOutput.queue, tcpOutput.writeId)), gs.IsTrue)
			tcpOutput.writeFile.WriteString("this is a test item")
			tcpOutput.writeFile.Close()
			tcpOutput.writeCheckpoint(tcpOutput.writeId, 10)
			tcpOutput.checkpointFile.Close()
			err = tcpOutput.readFromNextFile()
			buf := make([]byte, 4)
			n, err := tcpOutput.readFile.Read(buf)
			c.Expect(n, gs.Equals, 4)
			c.Expect("test", gs.Equals, string(buf))
			tcpOutput.writeFile.Close()
			tcpOutput.readFile.Close()
		})
	})

	c.Specify("TcpOutput", func() {
		origBaseDir := Globals().BaseDir
		Globals().BaseDir = tmpDir
		defer func() {
			Globals().BaseDir = origBaseDir
		}()

		tcpOutput := new(TcpOutput)
		tcpOutput.SetName("test")
		config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
		tcpOutput.Init(config)

		tickChan := make(chan time.Time)
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan)

		var wg sync.WaitGroup
		inChan := make(chan *PipelinePack, 1)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("writes out to the network", func() {
			inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
			inChanCall.Return(inChan)

			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				ch <- "ready"
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch // wait for server

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			outStr := "Write me out to the network"
			pack.Message.SetPayload(outStr)
			go func() {
				wg.Add(1)
				c.Expect(err, gs.IsNil)
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()

			inChan <- pack

			output := false
			for x := 0; x < 5; x++ {
				if fileExists(tcpOutput.checkpointFilename) {
					output = true
					break
				}
				time.Sleep(time.Duration(500) * time.Millisecond)
			}
			c.Expect(output, gs.Equals, true)

			close(inChan)
			wg.Wait() // wait for close to finish, prevents intermittent test failures

			matchBytes := make([]byte, 0, 1000)

			newpack := NewPipelinePack(nil)
			newpack.Message = msg
			newpack.Decoded = true
			newpack.Message.SetPayload(outStr)
			err = ProtobufEncodeMessage(newpack, &matchBytes)
			c.Expect(err, gs.IsNil)

			result = <-ch
			c.Expect(result, gs.Equals, string(matchBytes))
			id, offset, err := readCheckpoint(tcpOutput.checkpointFilename)
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(1))
			c.Expect(offset, gs.Equals, int64(115))
		})

		c.Specify("far end not initially listening", func() {
			inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
			inChanCall.Return(inChan)
			oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			outStr := "Write me out to the network"
			pack.Message.SetPayload(outStr)
			go func() {
				wg.Add(1)
				c.Expect(err, gs.IsNil)
				err = tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
			inChan <- pack

			output := false
			for x := 0; x < 5; x++ {
				if fileExists(getQueueFilename(tcpOutput.queue, tcpOutput.writeId)) {
					output = true
					time.Sleep(time.Duration(1) * time.Second) // give the send time to fail
					break
				}
				time.Sleep(time.Duration(500) * time.Millisecond)
			}
			c.Expect(output, gs.Equals, true)

			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch // wait for server

			close(inChan)
			wg.Wait() // wait for close to finish, prevents intermittent test failures

			matchBytes := make([]byte, 0, 1000)

			newpack := NewPipelinePack(nil)
			newpack.Message = msg
			newpack.Decoded = true
			newpack.Message.SetPayload(outStr)
			err = ProtobufEncodeMessage(newpack, &matchBytes)
			c.Expect(err, gs.IsNil)

			c.Expect(result, gs.Equals, string(matchBytes))
			id, offset, err := readCheckpoint(tcpOutput.checkpointFilename)
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(1))
			c.Expect(offset, gs.Equals, int64(115))
		})
	})
}
Пример #10
0
func UnixOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	c.Specify("A UnixOutput", func() {
		unixOutput := new(UdpOutput)
		config := unixOutput.ConfigStruct().(*UdpOutputConfig)

		oth := plugins_ts.NewOutputTestHelper(ctrl)
		encoder := new(plugins.PayloadEncoder)
		encoder.Init(new(plugins.PayloadEncoderConfig))

		inChan := make(chan *pipeline.PipelinePack, 1)
		rChan := make(chan *pipeline.PipelinePack, 1)
		msg := pipeline_ts.GetTestMessage()
		payload := "Write me out to the network."
		msg.SetPayload(payload)
		pack := pipeline.NewPipelinePack(rChan)
		pack.Message = msg

		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
		oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))

		c.Specify("using UDP", func() {
			addr := "@/heka/unix-out"
			config.Address = addr
			ch := make(chan string, 1)
			var wg sync.WaitGroup
			var rAddr net.Addr

			collectData := func() {
				conn, err := net.ListenPacket("unix", addr)
				if err != nil {
					ch <- err.Error()
					return
				}

				ch <- "ready"
				b := make([]byte, 1000)
				var n int
				n, rAddr, _ = conn.ReadFrom(b)
				ch <- string(b[:n])
				conn.Close()
			}

			go collectData()
			result := <-ch // Wait for server to be ready.
			c.Assume(result, gs.Equals, "ready")

			c.Specify("writes out to the network", func() {
				err := unixOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				close(inChan)
				wg.Wait()
			})

			c.Specify("uses the specified local address", func() {
				config.LocalAddress = "@/heka/unix-out"
				err := unixOutput.Init(config)
				c.Assume(err, gs.IsNil)

				wg.Add(1)
				go func() {
					err = unixOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()

				inChan <- pack
				result = <-ch

				c.Expect(result, gs.Equals, payload)
				c.Expect(rAddr.Network(), gs.Equals, "unix")
				c.Expect(rAddr.String(), gs.Equals, "@/heka/unix-out")
				close(inChan)
				wg.Wait()
			})
		})
	})
}
Пример #11
0
func NsqOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)
	var wg sync.WaitGroup

	errChan := make(chan error, 1)
	defer close(errChan)

	c.Specify("A nsq output", func() {
		output := new(NsqOutput)
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		config := output.ConfigStruct().(*NsqOutputConfig)
		config.Topic = "test"

		startOutput := func() {
			wg.Add(1)
			go func() {
				errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
		}

		var mockProducer *MockProducer
		output.newProducer = func(addr string, config *nsq.Config) (Producer, error) {
			mockProducer, _ = NewMockProducer(addr, config)
			return mockProducer, nil
		}

		msg := pipeline_ts.GetTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg

		c.Specify("requires at least one address", func() {
			err := output.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		config.Addresses = []string{"test.com:4150"}

		c.Specify("requires an encoder", func() {
			err := output.Init(config)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encoder().Return(nil)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("that is started", func() {
			encoder := new(plugins.PayloadEncoder)
			encoder.Init(&plugins.PayloadEncoderConfig{PrefixTs: false})
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))

			inChan := make(chan *pipeline.PipelinePack, 1)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)

			c.Specify("publishes a message with the configured topic", func() {
				err := output.Init(config)
				c.Assume(err, gs.IsNil)

				startOutput()

				inChan <- pack
				transaction := <-mockProducer.respChan
				c.Expect(transaction.Error, gs.IsNil)

				close(inChan)
				wg.Wait()
				c.Expect(<-errChan, gs.IsNil)
			})
		})

	})
}
Пример #12
0
func OutputSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	pConfig := pipeline.NewPipelineConfig(nil)
	inChan := make(chan *pipeline.PipelinePack, 1)
	timer := make(chan time.Time, 1)

	c.Specify("A SandboxOutput", func() {
		output := new(SandboxOutput)
		output.SetPipelineConfig(pConfig)
		conf := output.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/testsupport/output.lua"
		conf.ModuleDirectory = "../lua/modules;../lua/testsupport/modules"
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		data := "1376389920 debug id=2321 url=example.com item=1"
		pack.Message.SetPayload(data)

		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
		oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
		oth.MockOutputRunner.EXPECT().Ticker().Return(timer)

		c.Specify("writes a payload to file", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Assume(err, gs.IsNil)

			tmpFile, err := os.Open("output.lua.txt")
			defer tmpFile.Close()
			c.Assume(err, gs.IsNil)
			contents, err := ioutil.ReadAll(tmpFile)
			c.Assume(err, gs.IsNil)
			c.Expect(string(contents), gs.Equals, data)
			c.Expect(output.processMessageCount, gs.Equals, int64(1))
			c.Expect(output.processMessageSamples, gs.Equals, int64(1))
			c.Expect(output.processMessageFailures, gs.Equals, int64(0))
		})

		c.Specify("failure processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			pack.Message.SetPayload("FAILURE")
			pack.BufferedPack = true
			pack.DelivErrChan = make(chan error, 1)
			inChan <- pack
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Assume(err, gs.IsNil)
			c.Expect(output.processMessageFailures, gs.Equals, int64(1))
			err = <-pack.DelivErrChan
			c.Expect(err.Error(), gs.Equals, "failure message")
		})

		c.Specify("user abort processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			e := errors.New("FATAL: user abort")
			pack.Message.SetPayload("USERABORT")
			inChan <- pack
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err.Error(), gs.Equals, e.Error())
		})

		c.Specify("fatal error processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			pack.Message.SetPayload("FATAL")
			inChan <- pack
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("fatal error in timer_event", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			timer <- time.Now()
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("fatal error in shutdown timer_event", func() {
			conf.TimerEventOnShutdown = true
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
Пример #13
0
func TestSendMessage(t *testing.T) {
	ctrl := gomock.NewController(t)
	broker := sarama.NewMockBroker(t, 2)

	defer func() {
		broker.Close()
		ctrl.Finish()
	}()

	topic := "test"
	globals := DefaultGlobals()
	pConfig := NewPipelineConfig(globals)

	broker.SetHandlerByMap(map[string]sarama.MockResponse{
		"MetadataRequest": sarama.NewMockMetadataResponse(t).
			SetBroker(broker.Addr(), broker.BrokerID()).
			SetLeader(topic, 0, broker.BrokerID()),
		"ProduceRequest": sarama.NewMockProduceResponse(t),
	})

	ko := new(KafkaOutput)
	ko.SetPipelineConfig(pConfig)
	config := ko.ConfigStruct().(*KafkaOutputConfig)
	config.Addrs = append(config.Addrs, broker.Addr())
	config.Topic = topic
	err := ko.Init(config)
	if err != nil {
		t.Fatal(err)
	}
	oth := plugins_ts.NewOutputTestHelper(ctrl)
	encoder := new(plugins.PayloadEncoder)
	encoder.Init(encoder.ConfigStruct().(*plugins.PayloadEncoderConfig))

	inChan := make(chan *PipelinePack, 1)

	msg := pipeline_ts.GetTestMessage()
	pack := NewPipelinePack(pConfig.InputRecycleChan())
	pack.Message = msg

	inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
	inChanCall.Return(inChan)

	errChan := make(chan error)
	startOutput := func() {
		go func() {
			err := ko.Run(oth.MockOutputRunner, oth.MockHelper)
			errChan <- err
		}()
	}

	oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
	oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))

	outStr := "Write me out to the network"
	pack.Message.SetPayload(outStr)
	startOutput()

	msgcount := atomic.LoadInt64(&ko.processMessageCount)
	if msgcount != 0 {
		t.Errorf("Invalid starting processMessageCount %d", msgcount)
	}
	msgcount = atomic.LoadInt64(&ko.processMessageFailures)
	if msgcount != 0 {
		t.Errorf("Invalid starting processMessageFailures %d", msgcount)
	}

	inChan <- pack
	close(inChan)
	err = <-errChan
	if err != nil {
		t.Errorf("Error running output %s", err)
	}

	msgcount = atomic.LoadInt64(&ko.processMessageCount)
	if msgcount != 1 {
		t.Errorf("Invalid ending processMessageCount %d", msgcount)
	}
	msgcount = atomic.LoadInt64(&ko.processMessageFailures)
	if msgcount != 0 {
		t.Errorf("Invalid ending processMessageFailures %d", msgcount)
	}
}
Пример #14
0
func TcpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests")
	defer func() {
		ctrl.Finish()
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.Equals, nil)
	}()
	globals := DefaultGlobals()
	globals.BaseDir = tmpDir

	pConfig := NewPipelineConfig(globals)
	pConfig.RegisterDefault("HekaFramingSplitter")

	c.Specify("TcpOutput", func() {
		tcpOutput := new(TcpOutput)
		tcpOutput.SetName("test")
		config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
		tcpOutput.Init(config)

		tickChan := make(chan time.Time)
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan).AnyTimes()
		encoder := new(ProtobufEncoder)
		encoder.SetPipelineConfig(pConfig)
		encoder.Init(nil)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.QueueCursor = "queuecursor"

		outStr := "Write me out to the network"
		newpack := NewPipelinePack(nil)
		newpack.Message = msg
		newpack.Message.SetPayload(outStr)
		matchBytes, err := proto.Marshal(newpack.Message)
		c.Assume(err, gs.IsNil)
		pack.MsgBytes = matchBytes
		newpack.MsgBytes = matchBytes

		oth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

		c.Specify("doesn't use framing w/o ProtobufEncoder", func() {
			encoder := new(plugins.PayloadEncoder)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.IsNil)
			// We should fail if SetUseFraming is called since we didn't
			// EXPECT it.
		})

		c.Specify("doesn't use framing if config says not to", func() {
			useFraming := false
			config.UseFraming = &useFraming
			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.IsNil)
			// We should fail if SetUseFraming is called since we didn't
			// EXPECT it.
		})

		c.Specify("writes out to the network", func() {
			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				ch <- "ready"
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch // wait for server

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().SetUseFraming(true)
			err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)

			pack.Message.SetPayload(outStr)

			msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			err = tcpOutput.ProcessMessage(pack)
			c.Expect(err, gs.IsNil)
			result = <-ch

			msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(1))
			c.Expect(result, gs.Equals, string(matchBytes))

			tcpOutput.CleanUp()
		})

		c.Specify("far end not initially listening", func() {
			oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().SetUseFraming(true)
			err = tcpOutput.Prepare(oth.MockOutputRunner, oth.MockHelper)
			c.Assume(err, gs.IsNil)

			pack.Message.SetPayload(outStr)

			msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			err = tcpOutput.ProcessMessage(pack)
			_, ok := err.(RetryMessageError)
			c.Expect(ok, gs.IsTrue)
			msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			// After the message is queued start the collector. However, we
			// don't have a way guarantee a send attempt has already been made
			// and that we are actually exercising the retry code.
			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				ch <- "ready"
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch

			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)

			err = tcpOutput.ProcessMessage(pack)
			c.Expect(err, gs.IsNil)

			result = <-ch
			c.Expect(result, gs.Equals, string(matchBytes))
			msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(1))

			tcpOutput.CleanUp()
		})

		// c.Specify("Overload queue drops messages", func() {
		// 	config.QueueFullAction = "drop"
		// 	config.QueueMaxBufferSize = uint64(1)
		// 	use_framing := false
		// 	config.UseFraming = &use_framing
		// 	oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
		// 	oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)

		// 	err := tcpOutput.Init(config)
		// 	c.Expect(err, gs.IsNil)

		// 	startOutput()

		// 	inChan <- pack

		// 	dropcount := atomic.LoadInt64(&tcpOutput.dropMessageCount)

		// 	for x := 0; x < 5 && dropcount == 0; x++ {
		// 		dropcount = atomic.LoadInt64(&tcpOutput.dropMessageCount)
		// 		time.Sleep(time.Duration(100) * time.Millisecond)
		// 	}

		// 	c.Expect(dropcount, gs.Equals, int64(1))

		// 	close(inChan)
		// })

		// c.Specify("Overload queue shutdowns Heka", func() {
		// 	config.QueueFullAction = "shutdown"
		// 	config.QueueMaxBufferSize = uint64(1)
		// 	use_framing := false
		// 	config.UseFraming = &use_framing

		// 	oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
		// 	oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)

		// 	sigChan := globals.SigChan()

		// 	err := tcpOutput.Init(config)
		// 	c.Expect(err, gs.IsNil)

		// 	startOutput()

		// 	inChan <- pack
		// 	shutdownSignal := <-sigChan
		// 	c.Expect(shutdownSignal, gs.Equals, syscall.SIGINT)

		// 	close(inChan)
		// })

		// c.Specify("Overload queue blocks processing until packet is sent", func() {
		// 	config.QueueFullAction = "block"
		// 	config.QueueMaxBufferSize = uint64(1)
		// 	use_framing := false
		// 	config.UseFraming = &use_framing

		// 	oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes()
		// 	oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)

		// 	err := tcpOutput.Init(config)
		// 	c.Expect(err, gs.IsNil)

		// 	startOutput()

		// 	inChan <- pack

		// 	msgcount := atomic.LoadInt64(&tcpOutput.dropMessageCount)

		// 	for x := 0; x < 5 && msgcount == 0; x++ {
		// 		msgcount = atomic.LoadInt64(&tcpOutput.dropMessageCount)
		// 		time.Sleep(time.Duration(100) * time.Millisecond)
		// 	}

		// 	c.Expect(atomic.LoadInt64(&tcpOutput.dropMessageCount), gs.Equals, int64(0))
		// 	c.Expect(atomic.LoadInt64(&tcpOutput.processMessageCount), gs.Equals, int64(0))

		// 	close(inChan)
		// })

	})
}
Пример #15
0
func FileOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	c.Specify("A FileOutput", func() {
		fileOutput := new(FileOutput)

		tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
		tmpFilePath := fmt.Sprint(os.TempDir(), string(os.PathSeparator),
			tmpFileName)
		config := fileOutput.ConfigStruct().(*FileOutputConfig)
		config.Path = tmpFilePath

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		toString := func(outData interface{}) string {
			return string(*(outData.(*[]byte)))
		}

		c.Specify("correctly formats text output", func() {
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 20)

			c.Specify("by default", func() {
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				c.Expect(toString(&outData), gs.Equals, *msg.Payload+"\n")
			})

			c.Specify("w/ a prepended timestamp when specified", func() {
				fileOutput.prefix_ts = true
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				payload := *msg.Payload
				c.Expect(strContents, pipeline_ts.StringContains, payload)
				c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
			})

			c.Specify("even when payload is nil", func() {
				pack.Message.Payload = nil
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				strContents := toString(&outData)
				c.Expect(strContents, gs.Equals, "\n")
			})

			c.Specify("payload is nil and with a timestamp", func() {
				pack.Message.Payload = nil
				fileOutput.prefix_ts = true
				err = fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				c.Expect(strings.HasPrefix(strContents, todayStr), gs.IsTrue)
				c.Expect(strings.HasSuffix(strContents, " \n"), gs.IsTrue)
			})
		})

		c.Specify("correctly formats JSON output", func() {
			config.Format = "json"
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 200)

			c.Specify("when specified", func() {
				fileOutput.handleMessage(pack, &outData)
				msgJson, err := json.Marshal(pack.Message)
				c.Assume(err, gs.IsNil)
				c.Expect(toString(&outData), gs.Equals, string(msgJson)+"\n")
			})

			c.Specify("and with a timestamp", func() {
				fileOutput.prefix_ts = true
				fileOutput.handleMessage(pack, &outData)
				// Test will fail if date flips btn handleMessage call and
				// todayStr calculation... should be extremely rare.
				todayStr := time.Now().Format("[2006/Jan/02:")
				strContents := toString(&outData)
				msgJson, err := json.Marshal(pack.Message)
				c.Assume(err, gs.IsNil)
				c.Expect(strContents, pipeline_ts.StringContains, string(msgJson)+"\n")
				c.Expect(strContents, pipeline_ts.StringStartsWith, todayStr)
			})
		})

		c.Specify("correctly formats protocol buffer stream output", func() {
			config.Format = "protobufstream"
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			outData := make([]byte, 0, 200)

			c.Specify("when specified and timestamp ignored", func() {
				fileOutput.prefix_ts = true
				err := fileOutput.handleMessage(pack, &outData)
				c.Expect(err, gs.IsNil)
				b := []byte{30, 2, 8, uint8(proto.Size(pack.Message)), 31, 10, 16} // sanity check the header and the start of the protocol buffer
				c.Expect(bytes.Equal(b, outData[:len(b)]), gs.IsTrue)
			})
		})

		c.Specify("processes incoming messages", func() {
			err := fileOutput.Init(config)
			defer os.Remove(tmpFilePath)
			c.Assume(err, gs.IsNil)
			// Save for comparison.
			payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())

			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			wg.Add(1)
			go fileOutput.receiver(oth.MockOutputRunner, &wg)
			inChan <- pack
			close(inChan)
			outBatch := <-fileOutput.batchChan
			wg.Wait()
			c.Expect(string(outBatch), gs.Equals, payload)
		})

		c.Specify("Init halts if basedirectory is not writable", func() {
			tmpdir := filepath.Join(os.TempDir(), "tmpdir")
			err := os.MkdirAll(tmpdir, 0400)
			c.Assume(err, gs.IsNil)
			config.Path = tmpdir
			err = fileOutput.Init(config)
			c.Assume(err, gs.Not(gs.IsNil))
		})

		c.Specify("commits to a file", func() {
			outStr := "Write me out to the log file"
			outBytes := []byte(outStr)

			c.Specify("with default settings", func() {
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				contents, err := ioutil.ReadAll(tmpFile)
				c.Assume(err, gs.IsNil)
				c.Expect(string(contents), gs.Equals, outStr)
			})

			c.Specify("with different Perm settings", func() {
				config.Perm = "600"
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)

				// Start committer loop
				wg.Add(1)
				go fileOutput.committer(oth.MockOutputRunner, &wg)

				// Feed and close the batchChan
				go func() {
					fileOutput.batchChan <- outBytes
					_ = <-fileOutput.backChan // clear backChan to prevent blocking
					close(fileOutput.batchChan)
				}()

				wg.Wait()
				// Wait for the file close operation to happen.
				//for ; err == nil; _, err = fileOutput.file.Stat() {
				//}

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				fileInfo, err := tmpFile.Stat()
				c.Assume(err, gs.IsNil)
				fileMode := fileInfo.Mode()
				if runtime.GOOS == "windows" {
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
				} else {
					// 7 consecutive dashes implies no perms for group or other
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
				}
			})
		})

		c.Specify("honors folder_perm setting", func() {
			config.FolderPerm = "750"
			config.Path = filepath.Join(tmpFilePath, "subfile")
			err := fileOutput.Init(config)
			defer os.Remove(config.Path)
			c.Assume(err, gs.IsNil)

			fi, err := os.Stat(tmpFilePath)
			c.Expect(fi.IsDir(), gs.IsTrue)
			c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
		})

		c.Specify("that starts receiving w/ a flush interval", func() {
			config.FlushInterval = 100000000 // We'll trigger the timer manually.
			inChan := make(chan *PipelinePack)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			timerChan := make(chan time.Time)

			msg2 := pipeline_ts.GetTestMessage()
			pack2 := NewPipelinePack(pConfig.InputRecycleChan())
			pack2.Message = msg2

			recvWithConfig := func(config *FileOutputConfig) {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)
				wg.Add(1)
				go fileOutput.receiver(oth.MockOutputRunner, &wg)
				runtime.Gosched() // Yield so we can overwrite the timerChan.
				fileOutput.timerChan = timerChan
			}

			cleanUp := func() {
				close(inChan)
				wg.Done()
			}

			c.Specify("honors flush interval", func() {
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack
				select {
				case _ = <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}
				timerChan <- time.Now()
				select {
				case _ = <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval AND flush count", func() {
				config.FlushCount = 2
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				timerChan <- time.Now()
				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				inChan <- pack2
				runtime.Gosched()
				select {
				case <-fileOutput.batchChan:
				default:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval OR flush count", func() {
				config.FlushCount = 2
				config.FlushOperator = "OR"
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				default:
				}

				c.Specify("when interval triggers first", func() {
					timerChan <- time.Now()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})

				c.Specify("when count triggers first", func() {
					inChan <- pack2
					runtime.Gosched()
					select {
					case <-fileOutput.batchChan:
					default:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})
			})
		})
	})
}
Пример #16
0
func HttpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	var (
		reqMethod string
		reqBody   string
		reqHeader http.Header
		handleWg  sync.WaitGroup
		runWg     sync.WaitGroup
		delay     bool
	)

	handler := new(_test_handler)
	handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) {
		defer handleWg.Done()
		if delay {
			time.Sleep(time.Duration(2) * time.Millisecond)
		}
		// Capture request body for test assertions.
		p := make([]byte, req.ContentLength)
		_, err := req.Body.Read(p)
		c.Expect(err.Error(), gs.Equals, "EOF")
		reqBody = string(p)
		// Capture request method and headers for test assertions.
		reqMethod = req.Method
		reqHeader = req.Header
		// Respond with either a response body or a response code.
		if len(handler.respBody) > 0 {
			rw.Write([]byte(handler.respBody))
		} else {
			rw.WriteHeader(handler.respCode)
		}
		flusher := rw.(http.Flusher)
		flusher.Flush()
	}

	oth := ts.NewOutputTestHelper(ctrl)
	encoder := new(plugins.PayloadEncoder)
	encConfig := new(plugins.PayloadEncoderConfig)
	err := encoder.Init(encConfig)
	c.Expect(err, gs.IsNil)
	inChan := make(chan *pipeline.PipelinePack, 1)
	recycleChan := make(chan *pipeline.PipelinePack, 1)
	pack := pipeline.NewPipelinePack(recycleChan)
	pack.Message = pipeline_ts.GetTestMessage()

	c.Specify("An HttpOutput", func() {
		httpOutput := new(HttpOutput)
		config := httpOutput.ConfigStruct().(*HttpOutputConfig)

		c.Specify("barfs on bogus URLs", func() {
			config.Address = "one-two-three-four"
			err := httpOutput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("that is started", func() {
			server := httptest.NewServer(handler)
			defer server.Close()

			runOutput := func() {
				httpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				runWg.Done()
			}

			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			payload := "this is the payload"
			pack.Message.SetPayload(payload)
			oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(
				[]byte(payload), nil)
			config.Address = server.URL
			handler.respBody = "Response Body"

			c.Specify("makes http POST requests by default", func() {
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(reqMethod, gs.Equals, "POST")
			})

			c.Specify("makes http PUT requests", func() {
				config.Method = "put"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(reqMethod, gs.Equals, "PUT")
			})

			c.Specify("makes http GET requests", func() {
				config.Method = "get"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, "")
				c.Expect(reqMethod, gs.Equals, "GET")
			})

			c.Specify("correctly passes headers along", func() {
				config.Headers = http.Header{
					"One":  []string{"two", "three"},
					"Four": []string{"five", "six", "seven"},
				}
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"]))
				c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"]))
			})

			c.Specify("uses http auth when specified", func() {
				config.Username = "******"
				config.Password = "******"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				auth := reqHeader.Get("Authorization")
				c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue)
				decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:])
				c.Expect(err, gs.IsNil)
				c.Expect(string(decodedAuth), gs.Equals, "user:pass")
			})

			c.Specify("logs error responses", func() {
				handler.respBody = ""
				handler.respCode = 500
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)

				var errMsg string
				oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
					func(err error) {
						errMsg = err.Error()
					})
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(strings.HasPrefix(errMsg,
					"HTTP Error code returned: 500"), gs.IsTrue)
			})

			c.Specify("honors http timeout interval", func() {
				config.HttpTimeout = 1 // 1 millisecond
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)

				var errMsg string
				oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
					func(err error) {
						errMsg = err.Error()
					})
				delay = true
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(strings.Contains(errMsg, "use of closed network connection"),
					gs.IsTrue)
			})
		})
	})
}
Пример #17
0
func SmtpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	encoder := new(plugins.PayloadEncoder)
	econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
	econfig.AppendNewlines = false
	encoder.Init(econfig)

	c.Specify("A SmtpOutput", func() {
		smtpOutput := new(SmtpOutput)

		config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
		config.SendTo = []string{"root"}

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)
		runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
		runnerName.Return("SmtpOutput")
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
		encCall := oth.MockOutputRunner.EXPECT().Encode(pack)

		c.Specify("send email payload message", func() {
			err := smtpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			smtpOutput.sendFunction = testSendMail

			outStr := "Write me out to the network"
			pack.Message.SetPayload(outStr)
			encCall.Return(encoder.Encode(pack))
			wg.Add(1)
			go func() {
				smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
			inChan <- pack
			close(inChan)
			wg.Wait()
		})
	})

	c.Specify("SmtpOutput Message Body Encoding", func() {
		smtpOutput := new(SmtpOutput)
		chars := "123456789012345678901234567890123456789012345678901234567"
		charsE := "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3"
		examples := [][]string{
			{"Hello", "SGVsbG8="},
			{chars, charsE},
			{chars + chars, charsE + "\r\n" + charsE},
			{chars + chars + "Hello", charsE + "\r\n" + charsE + "\r\n" + "SGVsbG8="},
			{"", ""},
			{"1", "MQ=="},
		}
		for _, example := range examples {
			smtpOutput.encodeFullMsg([]byte(example[0]))
			c.Expect(string(smtpOutput.fullMsg), gs.Equals, example[1])
		}
	})

	// // Use this test with a real server
	// c.Specify("Real SmtpOutput output", func() {
	// 	smtpOutput := new(SmtpOutput)

	// 	config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
	// 	config.SendTo = []string{"root"}

	// 	msg := pipeline_ts.GetTestMessage()
	// 	pack := NewPipelinePack(pConfig.InputRecycleChan())
	// 	pack.Message = msg
	// 	pack.Decoded = true
	// 	inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
	// 	inChanCall.Return(inChan)
	// 	runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
	// 	runnerName.Return("SmtpOutput")
	// 	oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
	// 	encCall := oth.MockOutputRunner.EXPECT().Encode(pack)

	// 	c.Specify("send a real email essage", func() {

	// 		err := smtpOutput.Init(config)
	// 		c.Assume(err, gs.IsNil)

	// 		outStr := "Write me out to the network"
	// 		pack.Message.SetPayload(outStr)
	// 		encCall.Return(encoder.Encode(pack))
	// 		go func() {
	// 			wg.Add(1)
	// 			smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
	// 			wg.Done()
	// 		}()
	// 		inChan <- pack
	// 		time.Sleep(1000) // allow time for the message output
	// 		close(inChan)
	// 		wg.Wait()
	// // manually check the mail
	//	})
	// })
}
Пример #18
0
func FileOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	tmpFileName := fmt.Sprintf("fileoutput-test-%d", time.Now().UnixNano())
	tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)

	defer func() {
		ctrl.Finish()
		os.Remove(tmpFilePath)
	}()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	c.Specify("A FileOutput", func() {
		fileOutput := new(FileOutput)
		encoder := new(plugins.PayloadEncoder)
		encoder.Init(encoder.ConfigStruct())

		config := fileOutput.ConfigStruct().(*FileOutputConfig)
		config.Path = tmpFilePath

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.QueueCursor = "queuecursor"

		errChan := make(chan error, 1)

		c.Specify("w/ ProtobufEncoder", func() {
			encoder := new(ProtobufEncoder)
			encoder.SetPipelineConfig(pConfig)
			encoder.Init(nil)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)

			c.Specify("uses framing", func() {
				oth.MockOutputRunner.EXPECT().SetUseFraming(true)
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)
				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				wg.Add(1)
				go func() {
					err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()
				close(inChan)
				wg.Wait()
			})

			c.Specify("but not if config says not to", func() {
				useFraming := false
				config.UseFraming = &useFraming
				err := fileOutput.Init(config)
				defer os.Remove(tmpFilePath)
				c.Assume(err, gs.IsNil)
				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				wg.Add(1)
				go func() {
					err = fileOutput.Run(oth.MockOutputRunner, oth.MockHelper)
					c.Expect(err, gs.IsNil)
					wg.Done()
				}()
				close(inChan)
				wg.Wait()
				// We should fail if SetUseFraming is called since we didn't
				// EXPECT it.

			})
		})

		c.Specify("rotates files correctly", func() {
			config.Path = "%Y-%m-%d"
			config.RotationInterval = 24
			rotateChan := make(chan time.Time)
			closingChan := make(chan struct{})

			err := fileOutput.Init(config)
			c.Assume(err, gs.IsNil)

			defer fileOutput.file.Close()

			fileOutput.rotateChan = rotateChan
			fileOutput.closing = closingChan

			fileOutput.startRotateNotifier()

			committerChan := make(chan struct{})
			go func() {
				fileOutput.committer(oth.MockOutputRunner, errChan)
				close(committerChan)
			}()

			c.Assume(fileOutput.path, gs.Equals, time.Now().Format("2006-01-02"))

			futureDuration, _ := time.ParseDuration("24h")
			futureNow := time.Now().Add(futureDuration)

			rotateChan <- futureNow
			close(inChan)
			close(fileOutput.batchChan)
			<-committerChan

			c.Assume(fileOutput.path, gs.Equals, futureNow.Format("2006-01-02"))

		})

		c.Specify("processes incoming messages", func() {
			err := fileOutput.Init(config)
			c.Assume(err, gs.IsNil)
			fileOutput.file.Close()
			// Save for comparison.
			payload := fmt.Sprintf("%s\n", pack.Message.GetPayload())

			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			go fileOutput.receiver(oth.MockOutputRunner, errChan)
			inChan <- pack
			close(inChan)
			outBatch := <-fileOutput.batchChan
			c.Expect(string(outBatch.data), gs.Equals, payload)
			c.Expect(outBatch.cursor, gs.Equals, pack.QueueCursor)
		})

		c.Specify("commits to a file", func() {
			outStr := "Write me out to the log file"
			outBytes := []byte(outStr)
			batch := &outBatch{
				data:   outBytes,
				cursor: pack.QueueCursor,
			}

			oth.MockOutputRunner.EXPECT().UpdateCursor(pack.QueueCursor)

			c.Specify("with default settings", func() {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)

				// Start committer loop.
				go fileOutput.committer(oth.MockOutputRunner, errChan)

				// Feed and close the batchChan.
				go func() {
					fileOutput.batchChan <- batch
					_ = <-fileOutput.backChan // clear backChan to prevent blocking.
					close(fileOutput.batchChan)
				}()

				// Wait until we know processing has finished.
				<-fileOutput.closing

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				contents, err := ioutil.ReadAll(tmpFile)
				c.Assume(err, gs.IsNil)
				c.Expect(string(contents), gs.Equals, outStr)
			})

			c.Specify("with different Perm settings", func() {
				config.Perm = "600"
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)

				// Start committer loop.
				go fileOutput.committer(oth.MockOutputRunner, errChan)

				// Feed and close the batchChan.
				go func() {
					fileOutput.batchChan <- batch
					_ = <-fileOutput.backChan // clear backChan to prevent blocking.
					close(fileOutput.batchChan)
				}()

				// Wait until we know processing has finished.
				<-fileOutput.closing

				tmpFile, err := os.Open(tmpFilePath)
				defer tmpFile.Close()
				c.Assume(err, gs.IsNil)
				fileInfo, err := tmpFile.Stat()
				c.Assume(err, gs.IsNil)
				fileMode := fileInfo.Mode()
				if runtime.GOOS == "windows" {
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-rw-rw-rw-")
				} else {
					// 7 consecutive dashes implies no perms for group or other.
					c.Expect(fileMode.String(), pipeline_ts.StringContains, "-------")
				}
			})
		})

		if runtime.GOOS != "windows" {
			if u, err := user.Current(); err != nil && u.Uid != "0" {
				c.Specify("Init halts if basedirectory is not writable", func() {
					tmpdir := filepath.Join(os.TempDir(), "tmpdir")
					err := os.MkdirAll(tmpdir, 0400)
					c.Assume(err, gs.IsNil)
					config.Path = filepath.Join(tmpdir, "out.txt")
					err = fileOutput.Init(config)
					c.Assume(err, gs.Not(gs.IsNil))
					os.RemoveAll(tmpdir)
				})
			}

			c.Specify("honors folder_perm setting", func() {
				config.FolderPerm = "750"
				subdir := filepath.Join(os.TempDir(), "subdir")
				config.Path = filepath.Join(subdir, "out.txt")
				err := fileOutput.Init(config)
				defer os.RemoveAll(subdir)
				c.Assume(err, gs.IsNil)

				fi, err := os.Stat(subdir)
				c.Expect(fi.IsDir(), gs.IsTrue)
				c.Expect(fi.Mode().Perm(), gs.Equals, os.FileMode(0750))
			})
		}

		c.Specify("that starts receiving w/ a flush interval", func() {
			config.FlushInterval = 100000000 // We'll trigger the timer manually.
			inChan := make(chan *PipelinePack)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			timerChan := make(chan time.Time)

			msg2 := pipeline_ts.GetTestMessage()
			msg2.SetPayload("MESSAGE 2")
			pack2 := NewPipelinePack(pConfig.InputRecycleChan())
			pack2.Message = msg2

			recvWithConfig := func(config *FileOutputConfig) {
				err := fileOutput.Init(config)
				c.Assume(err, gs.IsNil)

				fileOutput.timerChan = timerChan
				go fileOutput.receiver(oth.MockOutputRunner, errChan)
				runtime.Gosched() // Yield so receiver will start.
			}

			cleanUp := func() {
				close(inChan)
				fileOutput.file.Close()
			}

			c.Specify("honors flush interval", func() {
				oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				after := time.After(100 * time.Millisecond)
				select {
				case _ = <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				case <-after:
				}

				timerChan <- time.Now()
				after = time.After(100 * time.Millisecond)
				select {
				case _ = <-fileOutput.batchChan:
				case <-after:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval AND flush count", func() {
				oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
				oth.MockOutputRunner.EXPECT().Encode(pack2).Return(encoder.Encode(pack2))
				config.FlushCount = 2
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				after := time.After(100 * time.Millisecond)
				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				case <-after:
				}

				timerChan <- time.Now()
				after = time.After(100 * time.Millisecond)
				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				case <-after:
				}

				after = time.After(100 * time.Millisecond)
				inChan <- pack2
				select {
				case <-fileOutput.batchChan:
				case <-after:
					c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
				}
			})

			c.Specify("honors flush interval OR flush count", func() {
				oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(encoder.Encode(pack))
				config.FlushCount = 2
				config.FlushOperator = "OR"
				recvWithConfig(config)
				defer cleanUp()
				inChan <- pack

				after := time.After(100 * time.Millisecond)
				select {
				case <-fileOutput.batchChan:
					c.Expect("", gs.Equals, "fileOutput.batchChan should NOT have fired yet")
				case <-after:
				}

				c.Specify("when interval triggers first", func() {
					timerChan <- time.Now()
					after = time.After(100 * time.Millisecond)
					select {
					case <-fileOutput.batchChan:
					case <-after:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})

				c.Specify("when count triggers first", func() {
					out, err := encoder.Encode(pack2)
					oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(out, err)
					inChan <- pack2
					after = time.After(100 * time.Millisecond)
					select {
					case <-fileOutput.batchChan:
					case <-after:
						c.Expect("", gs.Equals, "fileOutput.batchChan SHOULD have fired by now")
					}
				})
			})
		})
	})
}