Esempio n. 1
0
func StatsdInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
	ith.PackSupply = make(chan *PipelinePack, 1)

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, input runner, and stat accumulator
	ith.MockHelper = NewMockPluginHelper(ctrl)
	ith.MockInputRunner = NewMockInputRunner(ctrl)
	mockStatAccum := NewMockStatAccumulator(ctrl)

	c.Specify("A StatsdInput", func() {
		statsdInput := StatsdInput{}
		config := statsdInput.ConfigStruct().(*StatsdInputConfig)

		config.Address = ith.AddrStr
		err := statsdInput.Init(config)
		c.Assume(err, gs.IsNil)
		realListener := statsdInput.listener
		c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()
		mockListener := pipeline_ts.NewMockConn(ctrl)
		statsdInput.listener = mockListener

		ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil)
		mockListener.EXPECT().Close()
		mockListener.EXPECT().SetReadDeadline(gomock.Any())

		c.Specify("sends a Stat to the StatAccumulator", func() {
			statName := "sample.count"
			statVal := 303
			msg := fmt.Sprintf("%s:%d|c\n", statName, statVal)
			expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)}
			mockStatAccum.EXPECT().DropStat(expected).Return(true)
			readCall := mockListener.EXPECT().Read(make([]byte, 512))
			readCall.Return(len(msg), nil)
			readCall.Do(func(msgBytes []byte) {
				copy(msgBytes, []byte(msg))
				statsdInput.Stop()
			})
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()
			wg.Wait()
		})
	})
}
Esempio n. 2
0
func TestMatchers(t *testing.T) {
	type e interface{}
	type testCase struct {
		matcher gomock.Matcher
		yes, no []e
	}
	tests := []testCase{
		testCase{gomock.Any(), []e{3, nil, "foo"}, nil},
		testCase{gomock.Eq(4), []e{4}, []e{3, "blah", nil, int64(4)}},
		testCase{gomock.Nil(),
			[]e{nil, (error)(nil), (chan bool)(nil), (*int)(nil)},
			[]e{"", 0, make(chan bool), errors.New("err"), new(int)}},
		testCase{gomock.Not(gomock.Eq(4)), []e{3, "blah", nil, int64(4)}, []e{4}},
	}
	for i, test := range tests {
		for _, x := range test.yes {
			if !test.matcher.Matches(x) {
				t.Errorf(`test %d: "%v %s" should be true.`, i, x, test.matcher)
			}
		}
		for _, x := range test.no {
			if test.matcher.Matches(x) {
				t.Errorf(`test %d: "%v %s" should be false.`, i, x, test.matcher)
			}
		}
	}
}
Esempio n. 3
0
func TestSocketListenConfig(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()
	mckStat := NewMockStatistician(mockCtrl)
	mckListenerConfig := NewMockListenerConfig(mockCtrl)

	app := NewApplication()
	app.hostname = "example.org"
	app.SetLogger(mckLogger)
	app.SetMetrics(mckStat)

	sh := NewSocketHandler()
	sh.setApp(app)

	// Should forward Listen errors.
	listenErr := errors.New("splines not reticulated")
	mckListenerConfig.EXPECT().Listen().Return(nil, listenErr)
	if err := sh.listenWithConfig(mckListenerConfig); err != listenErr {
		t.Errorf("Wrong error: got %#v; want %#v", err, listenErr)
	}

	// Should use the wss:// scheme if UseTLS returns true.
	ml := newMockListener(netAddr{"test", "[::1]:8080"})
	gomock.InOrder(
		mckListenerConfig.EXPECT().Listen().Return(ml, nil),
		mckListenerConfig.EXPECT().UseTLS().Return(true),
		mckListenerConfig.EXPECT().GetMaxConns().Return(1),
	)
	if err := sh.listenWithConfig(mckListenerConfig); err != nil {
		t.Errorf("Error setting listener: %s", err)
	}
	if maxConns := sh.MaxConns(); maxConns != 1 {
		t.Errorf("Mismatched maximum connection count: got %d; want 1",
			maxConns)
	}
	expectedURL := "wss://example.org:8080"
	if url := sh.URL(); url != expectedURL {
		t.Errorf("Mismatched handler URL: got %q; want %q",
			url, expectedURL)
	}
}
Esempio n. 4
0
func UdpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	c.Specify("A UdpInput", func() {
		udpInput := UdpInput{}
		config := &UdpInputConfig{}

		mbytes, _ := proto.Marshal(ith.Msg)
		header := &message.Header{}
		header.SetMessageLength(uint32(len(mbytes)))
		hbytes, _ := proto.Marshal(header)
		buf := encodeMessage(hbytes, mbytes)

		bytesChan := make(chan []byte, 1)

		ith.MockInputRunner.EXPECT().Name().Return("mock_name")
		ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
		ith.MockSplitterRunner.EXPECT().GetRemainingData().AnyTimes()
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

		splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
			nil).AnyTimes()
		splitCall.Do(func(conn net.Conn, del Deliverer) {
			recd := make([]byte, 65536)
			n, _ := conn.Read(recd)
			recd = recd[:n]
			bytesChan <- recd
		})

		c.Specify("using a udp address", func() {
			ith.AddrStr = "localhost:55565"
			ith.ResolvedAddrStr = "127.0.0.1:55565"
			config.Net = "udp"
			config.Address = ith.AddrStr

			err := udpInput.Init(config)
			c.Assume(err, gs.IsNil)
			realListener := (udpInput.listener).(*net.UDPConn)
			c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)

			c.Specify("passes the connection to SplitStream", func() {
				go udpInput.Run(ith.MockInputRunner, ith.MockHelper)

				conn, err := net.Dial("udp", ith.AddrStr)
				c.Assume(err, gs.IsNil)
				_, err = conn.Write(buf)
				c.Assume(err, gs.IsNil)
				conn.Close()

				recd := <-bytesChan
				c.Expect(string(recd), gs.Equals, string(buf))
				udpInput.Stop()
			})
		})

		if runtime.GOOS != "windows" {
			c.Specify("using a unix datagram socket", func() {
				tmpDir, err := ioutil.TempDir("", "heka-socket")
				c.Assume(err, gs.IsNil)
				unixPath := filepath.Join(tmpDir, "unixgram-socket")
				ith.AddrStr = unixPath
				config.Net = "unixgram"
				config.Address = ith.AddrStr

				err = udpInput.Init(config)
				c.Assume(err, gs.IsNil)
				realListener := (udpInput.listener).(*net.UnixConn)
				c.Expect(realListener.LocalAddr().String(), gs.Equals, unixPath)

				c.Specify("passes the socket to SplitStream", func() {
					go udpInput.Run(ith.MockInputRunner, ith.MockHelper)

					unixAddr, err := net.ResolveUnixAddr("unixgram", unixPath)
					c.Assume(err, gs.IsNil)
					conn, err := net.DialUnix("unixgram", nil, unixAddr)
					c.Assume(err, gs.IsNil)
					_, err = conn.Write(buf)
					c.Assume(err, gs.IsNil)
					conn.Close()

					recd := <-bytesChan
					c.Expect(string(recd), gs.Equals, string(buf))
					udpInput.Stop()
				})
			})
		}
	})
}
Esempio n. 5
0
func TestReceivePayloadMessage(t *testing.T) {
	b1 := sarama.NewMockBroker(t, 1)
	b2 := sarama.NewMockBroker(t, 2)
	ctrl := gomock.NewController(t)
	tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests")
	if tmpErr != nil {
		t.Errorf("Unable to create a temporary directory: %s", tmpErr)
	}

	defer func() {
		if err := os.RemoveAll(tmpDir); err != nil {
			t.Errorf("Cleanup failed: %s", err)
		}
		ctrl.Finish()
	}()

	topic := "test"
	mdr := new(sarama.MetadataResponse)
	mdr.AddBroker(b2.Addr(), b2.BrokerID())
	mdr.AddTopicPartition(topic, 0, 2)
	b1.Returns(mdr)

	or := new(sarama.OffsetResponse)
	or.AddTopicPartition(topic, 0, 0)
	b2.Returns(or)

	fr := new(sarama.FetchResponse)
	fr.AddMessage(topic, 0, nil, sarama.ByteEncoder([]byte{0x41, 0x42}), 0)
	b2.Returns(fr)

	pConfig := NewPipelineConfig(nil)
	pConfig.Globals.BaseDir = tmpDir
	ki := new(KafkaInput)
	ki.SetName(topic)
	ki.SetPipelineConfig(pConfig)
	config := ki.ConfigStruct().(*KafkaInputConfig)
	config.Addrs = append(config.Addrs, b1.Addr())
	config.Topic = topic

	ith := new(plugins_ts.InputTestHelper)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)

	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	err := ki.Init(config)
	if err != nil {
		t.Fatalf("%s", err)
	}

	ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
	ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)

	decChan := make(chan func(*PipelinePack), 1)
	decCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
	decCall.Do(func(dec func(pack *PipelinePack)) {
		decChan <- dec
	})

	bytesChan := make(chan []byte, 1)
	splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil)
	splitCall.Do(func(recd []byte, del Deliverer) {
		bytesChan <- recd
	})

	errChan := make(chan error)
	go func() {
		errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper)
	}()

	recd := <-bytesChan
	if string(recd) != "AB" {
		t.Errorf("Invalid Payload Expected: AB received: %s", string(recd))
	}

	packDec := <-decChan
	packDec(ith.Pack)
	if ith.Pack.Message.GetType() != "heka.kafka" {
		t.Errorf("Invalid Type %s", ith.Pack.Message.GetType())
	}

	// There is a hang on the consumer close with the mock broker
	// closing the brokers before the consumer works around the issue
	// and is good enough for this test.
	b1.Close()
	b2.Close()

	ki.Stop()
	err = <-errChan
	if err != nil {
		t.Fatal(err)
	}

	filename := filepath.Join(tmpDir, "kafka", "test.test.0.offset.bin")
	if o, err := readCheckpoint(filename); err != nil {
		t.Errorf("Could not read the checkpoint file: %s", filename)
	} else {
		if o != 1 {
			t.Errorf("Incorrect offset Expected: 1 Received: %d", o)
		}
	}
}
Esempio n. 6
0
func FilePollingInputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	tmpFileName := fmt.Sprintf("filepollinginput-test-%d", time.Now().UnixNano())
	tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)

	defer func() {
		ctrl.Finish()
		os.Remove(tmpFilePath)
	}()

	pConfig := NewPipelineConfig(nil)
	var wg sync.WaitGroup
	errChan := make(chan error, 1)
	bytesChan := make(chan []byte, 1)
	tickChan := make(chan time.Time)

	retPackChan := make(chan *PipelinePack, 2)
	defer close(retPackChan)

	c.Specify("A FilePollingInput", func() {
		input := new(FilePollingInput)

		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
		ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

		config := input.ConfigStruct().(*FilePollingInputConfig)
		config.FilePath = tmpFilePath

		startInput := func(msgCount int) {
			wg.Add(1)
			go func() {
				errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
				wg.Done()
			}()
		}

		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
		ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

		c.Specify("gets updated information when reading a file", func() {

			err := input.Init(config)
			c.Assume(err, gs.IsNil)

			ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
				nil).Return(io.EOF).Times(2)
			splitCall.Do(func(f *os.File, del Deliverer) {
				fBytes, err := ioutil.ReadAll(f)
				if err != nil {
					fBytes = []byte(err.Error())
				}
				bytesChan <- fBytes
			})

			startInput(2)

			f, err := os.Create(tmpFilePath)
			c.Expect(err, gs.IsNil)

			_, err = f.Write([]byte("test1"))
			c.Expect(err, gs.IsNil)
			c.Expect(f.Close(), gs.IsNil)

			tickChan <- time.Now()

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "test1")

			f, err = os.Create(tmpFilePath)
			c.Expect(err, gs.IsNil)

			_, err = f.Write([]byte("test2"))
			c.Expect(err, gs.IsNil)
			c.Expect(f.Close(), gs.IsNil)

			tickChan <- time.Now()
			msgBytes = <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "test2")

			input.Stop()
			wg.Wait()
			c.Expect(<-errChan, gs.IsNil)
		})

	})

}
Esempio n. 7
0
func TestReceiveProtobufMessage(t *testing.T) {
	broker := sarama.NewMockBroker(t, 2)
	ctrl := gomock.NewController(t)
	tmpDir, tmpErr := ioutil.TempDir("", "kafkainput-tests")
	if tmpErr != nil {
		t.Errorf("Unable to create a temporary directory: %s", tmpErr)
	}

	defer func() {
		if err := os.RemoveAll(tmpDir); err != nil {
			t.Errorf("Cleanup failed: %s", err)
		}
		ctrl.Finish()
	}()

	topic := "test"
	mockFetchResponse := sarama.NewMockFetchResponse(t, 1)
	mockFetchResponse.SetMessage(topic, 0, 0, sarama.ByteEncoder([]byte{0x41, 0x42}))

	broker.SetHandlerByMap(map[string]sarama.MockResponse{
		"MetadataRequest": sarama.NewMockMetadataResponse(t).
			SetBroker(broker.Addr(), broker.BrokerID()).
			SetLeader(topic, 0, broker.BrokerID()),
		"OffsetRequest": sarama.NewMockOffsetResponse(t).
			SetOffset(topic, 0, sarama.OffsetOldest, 0).
			SetOffset(topic, 0, sarama.OffsetNewest, 2),
		"FetchRequest": mockFetchResponse,
	})

	pConfig := NewPipelineConfig(nil)
	pConfig.Globals.BaseDir = tmpDir
	ki := new(KafkaInput)
	ki.SetName(topic)
	ki.SetPipelineConfig(pConfig)
	config := ki.ConfigStruct().(*KafkaInputConfig)
	config.Addrs = append(config.Addrs, broker.Addr())
	config.Topic = topic

	ith := new(plugins_ts.InputTestHelper)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)

	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

	err := ki.Init(config)
	if err != nil {
		t.Fatalf("%s", err)
	}

	ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
	ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
	ith.MockSplitterRunner.EXPECT().Done()

	bytesChan := make(chan []byte, 1)
	splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(), nil)
	splitCall.Do(func(recd []byte, del Deliverer) {
		bytesChan <- recd
	})

	errChan := make(chan error)
	go func() {
		errChan <- ki.Run(ith.MockInputRunner, ith.MockHelper)
	}()

	recd := <-bytesChan
	if string(recd) != "AB" {
		t.Errorf("Invalid MsgBytes Expected: AB received: %s", string(recd))
	}

	// There is a hang on the consumer close with the mock broker
	// closing the brokers before the consumer works around the issue
	// and is good enough for this test.
	broker.Close()

	ki.Stop()
	err = <-errChan
	if err != nil {
		t.Fatal(err)
	}
}
Esempio n. 8
0
func TcpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	tmpDir, tmpErr := ioutil.TempDir("", "tcp-tests")
	defer func() {
		ctrl.Finish()
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.Equals, nil)
	}()
	globals := DefaultGlobals()
	globals.BaseDir = tmpDir

	pConfig := NewPipelineConfig(globals)
	pConfig.RegisterDefault("HekaFramingSplitter")

	c.Specify("TcpOutput", func() {
		tcpOutput := new(TcpOutput)
		tcpOutput.SetName("test")
		config := tcpOutput.ConfigStruct().(*TcpOutputConfig)
		tcpOutput.Init(config)

		tickChan := make(chan time.Time)
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		oth.MockOutputRunner.EXPECT().Ticker().Return(tickChan).AnyTimes()
		encoder := new(ProtobufEncoder)
		encoder.SetPipelineConfig(pConfig)
		encoder.Init(nil)

		inChan := make(chan *PipelinePack, 1)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		outStr := "Write me out to the network"
		newpack := NewPipelinePack(nil)
		newpack.Message = msg
		newpack.Decoded = true
		newpack.Message.SetPayload(outStr)
		matchBytes, err := proto.Marshal(newpack.Message)
		c.Expect(err, gs.IsNil)

		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)

		errChan := make(chan error)
		startOutput := func() {
			go func() {
				oth.MockHelper.EXPECT().PipelineConfig().Return(pConfig).AnyTimes()
				oth.MockOutputRunner.EXPECT().Name().Return("TcpOutput")
				err := tcpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()
		}

		c.Specify("doesn't use framing w/o ProtobufEncoder", func() {
			encoder := new(plugins.PayloadEncoder)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			close(inChan)
			startOutput()
			err = <-errChan
			c.Expect(err, gs.IsNil)
			// We should fail if SetUseFraming is called since we didn't
			// EXPECT it.
		})

		c.Specify("doesn't use framing if config says not to", func() {
			useFraming := false
			config.UseFraming = &useFraming
			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			close(inChan)
			startOutput()
			err = <-errChan
			c.Expect(err, gs.IsNil)
			// We should fail if SetUseFraming is called since we didn't
			// EXPECT it.
		})

		c.Specify("writes out to the network", func() {
			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				ch <- "ready"
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch // wait for server

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().SetUseFraming(true)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()

			pack.Message.SetPayload(outStr)
			startOutput()

			msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			inChan <- pack
			result = <-ch

			msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(1))
			c.Expect(result, gs.Equals, string(matchBytes))

			close(inChan)
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})

		c.Specify("far end not initially listening", func() {
			oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()

			err := tcpOutput.Init(config)
			c.Assume(err, gs.IsNil)

			pack.Message.SetPayload(outStr)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().SetUseFraming(true)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().UsesFraming().Return(false).AnyTimes()

			startOutput()
			msgcount := atomic.LoadInt64(&tcpOutput.processMessageCount)
			c.Expect(msgcount, gs.Equals, int64(0))

			inChan <- pack

			for x := 0; x < 5 && msgcount == 0; x++ {
				msgcount = atomic.LoadInt64(&tcpOutput.processMessageCount)
				time.Sleep(time.Duration(100) * time.Millisecond)
			}

			// After the message is queued start the collector. However, we
			// don't have a way guarantee a send attempt has already been made
			// and that we are actually exercising the retry code.
			collectData := func(ch chan string) {
				ln, err := net.Listen("tcp", "localhost:9125")
				if err != nil {
					ch <- err.Error()
					return
				}
				conn, err := ln.Accept()
				if err != nil {
					ch <- err.Error()
					return
				}
				b := make([]byte, 1000)
				n, _ := conn.Read(b)
				ch <- string(b[0:n])
				conn.Close()
				ln.Close()
			}
			ch := make(chan string, 1) // don't block on put
			go collectData(ch)
			result := <-ch
			c.Expect(result, gs.Equals, string(matchBytes))

			close(inChan)
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Overload queue drops messages", func() {
			config.QueueFullAction = "drop"
			config.QueueMaxBufferSize = uint64(1)
			use_framing := false
			config.UseFraming = &use_framing
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)

			err := tcpOutput.Init(config)
			c.Expect(err, gs.IsNil)

			startOutput()

			inChan <- pack

			dropcount := atomic.LoadInt64(&tcpOutput.dropMessageCount)

			for x := 0; x < 5 && dropcount == 0; x++ {
				dropcount = atomic.LoadInt64(&tcpOutput.dropMessageCount)
				time.Sleep(time.Duration(100) * time.Millisecond)
			}

			c.Expect(dropcount, gs.Equals, int64(1))

			close(inChan)
		})

		c.Specify("Overload queue shutdowns Heka", func() {
			config.QueueFullAction = "shutdown"
			config.QueueMaxBufferSize = uint64(1)
			use_framing := false
			config.UseFraming = &use_framing

			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))
			oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)

			sigChan := globals.SigChan()

			err := tcpOutput.Init(config)
			c.Expect(err, gs.IsNil)

			startOutput()

			inChan <- pack
			shutdownSignal := <-sigChan
			c.Expect(shutdownSignal, gs.Equals, syscall.SIGINT)

			close(inChan)
		})

		c.Specify("Overload queue blocks processing until packet is sent", func() {
			config.QueueFullAction = "block"
			config.QueueMaxBufferSize = uint64(1)
			use_framing := false
			config.UseFraming = &use_framing

			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack)).AnyTimes()
			oth.MockOutputRunner.EXPECT().LogError(QueueIsFull)

			err := tcpOutput.Init(config)
			c.Expect(err, gs.IsNil)

			startOutput()

			inChan <- pack

			msgcount := atomic.LoadInt64(&tcpOutput.dropMessageCount)

			for x := 0; x < 5 && msgcount == 0; x++ {
				msgcount = atomic.LoadInt64(&tcpOutput.dropMessageCount)
				time.Sleep(time.Duration(100) * time.Millisecond)
			}

			c.Expect(atomic.LoadInt64(&tcpOutput.dropMessageCount), gs.Equals, int64(0))
			c.Expect(atomic.LoadInt64(&tcpOutput.processMessageCount), gs.Equals, int64(0))

			close(inChan)
		})

	})
}
Esempio n. 9
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	pConfig := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network.
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// Set up mock helper, runner, and pack supply channel.
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: pConfig}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs.
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
				ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
					ith.MockSplitterRunner)
				ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
				ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
				ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

				getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
					gomock.Any()).Times(numLines)
				line := "boo hoo foo foo"
				getRecCall.Return(len(line), []byte(line), nil)
				getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
				getRecCall.Return(0, make([]byte, 0), io.EOF)

				deliverChan := make(chan []byte, 1)
				deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
					ith.MockDeliverer).Times(numLines)
				deliverCall.Do(func(record []byte, del Deliverer) {
					deliverChan <- record
				})

				ith.MockDeliverer.EXPECT().Done()

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				dur, _ := time.ParseDuration("5s")
				timeout := time.After(dur)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case record := <-deliverChan:
						c.Expect(string(record), gs.Equals, line)
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Esempio n. 10
0
func TestSocketInvalidOrigin(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()
	mckStat := NewMockStatistician(mockCtrl)
	mckStat.EXPECT().Increment("client.socket.connect").AnyTimes()
	mckStat.EXPECT().Timer("client.socket.lifespan", gomock.Any()).AnyTimes()
	mckStat.EXPECT().Increment("client.socket.disconnect").AnyTimes()
	mckStore := NewMockStore(mockCtrl)
	mckRouter := NewMockRouter(mockCtrl)

	testWithOrigin := func(allowedOrigins []string,
		config *websocket.Config) error {

		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)
		app.SetStore(mckStore)
		app.SetRouter(mckRouter)

		sh := NewSocketHandler()
		defer sh.Close()
		sh.setApp(app)
		sh.setOrigins(allowedOrigins)
		pipe := newPipeListener()
		if err := sh.listenWithConfig(listenerConfig{listener: pipe}); err != nil {
			return err
		}
		sh.server = newServeWaiter(&http.Server{Handler: sh.ServeMux()})
		app.SetSocketHandler(sh)

		errChan := make(chan error, 1)
		go sh.Start(errChan)

		conn, err := dialSocketListener(pipe, config)
		if err != nil {
			return err
		}
		conn.Close()
		return nil
	}

	location := &url.URL{Scheme: "https", Host: "example.com"}
	badURL := &url.URL{Scheme: "!@#$", Host: "^&*-"}

	tests := []struct {
		name           string
		allowedOrigins []string
		config         *websocket.Config
		err            error
	}{
		{
			"Should allow all origins if none are specified",
			nil,
			&websocket.Config{
				Version:  websocket.ProtocolVersionHybi13,
				Location: location,
				Origin:   location,
			},
			nil,
		},
		{
			"Should match multiple origins",
			[]string{"https://example.com", "https://example.org", "https://example.net"},
			&websocket.Config{
				Version:  websocket.ProtocolVersionHybi13,
				Location: location,
				Origin:   &url.URL{Scheme: "https", Host: "example.net"},
			},
			nil,
		},
		{
			"Should reject mismatched origins",
			[]string{"https://example.com"},
			&websocket.Config{
				Version:  websocket.ProtocolVersionHybi13,
				Location: location,
				Origin:   &url.URL{Scheme: "http", Host: "example.org"},
			},
			websocket.ErrBadStatus, // 403 status code.
		},
		{
			"Should reject malformed origins if some are specified",
			[]string{"https://example.com"},
			&websocket.Config{
				Version:  websocket.ProtocolVersionHybi13,
				Location: location,
				Origin:   badURL, // Malformed URL.
			},
			websocket.ErrBadStatus, // 403 status code.
		},
		{
			"Should ignore malformed origins if none are specified",
			nil,
			&websocket.Config{
				Version:  websocket.ProtocolVersionHybi13,
				Location: location,
				Origin:   badURL,
			},
			nil,
		},
		{
			"Should allow empty origins if none are specified",
			nil,
			&websocket.Config{
				Version:  websocket.ProtocolVersionHybi13,
				Location: location,
				Origin:   &url.URL{},
			},
			nil,
		},
		{
			"Should reject empty origins if some are specified",
			[]string{"https://example.com"},
			&websocket.Config{
				Version:  websocket.ProtocolVersionHybi13,
				Location: location,
				Origin:   &url.URL{},
			},
			websocket.ErrBadStatus,
		},
	}
	for _, test := range tests {
		err := testWithOrigin(test.allowedOrigins, test.config)
		if err != test.err {
			t.Errorf("On test %s, got %#v; want %#v", test.name, err, test.err)
		}
	}
}
Esempio n. 11
0
func SplitterRunnerSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	srConfig := CommonSplitterConfig{}

	c.Specify("A SplitterRunner w/ HekaFramingSplitter", func() {
		splitter := &HekaFramingSplitter{}
		config := splitter.ConfigStruct().(*HekaFramingSplitterConfig)
		useMsgBytes := true
		srConfig.UseMsgBytes = &useMsgBytes
		sr := NewSplitterRunner("HekaFramingSplitter", splitter, srConfig)
		splitter.SetSplitterRunner(sr)

		err := splitter.Init(config)
		c.Assume(err, gs.IsNil)

		b, err := ioutil.ReadFile(filepath.Join(".", "testsupport", "multi.dat"))
		c.Assume(err, gs.IsNil)
		reader := makeMockReader(b)

		c.Specify("correctly handles data at EOF", func() {
			count, errCount, bytesRead, foundEOFCount,
				remainingDataLength, finalRecordLength,
				eofRecordLength := readRecordsFromStream(sr, reader, true)

			c.Expect(errCount, gs.Equals, 0)
			c.Expect(count, gs.Equals, 50)
			c.Expect(foundEOFCount, gs.Equals, 50)
			c.Expect(remainingDataLength, gs.Equals, 0)
			c.Expect(finalRecordLength, gs.Equals, 215)
			c.Expect(eofRecordLength, gs.Equals, 0)
			c.Expect(bytesRead, gs.Equals, len(b))
		})

		c.Specify("correctly splits & unframes a protobuf stream", func() {
			ir := NewMockInputRunner(ctrl)
			sr.SetInputRunner(ir)
			recycleChan := make(chan *PipelinePack, 1)
			pack := NewPipelinePack(recycleChan)
			recycleChan <- pack
			numRecs := 50
			ir.EXPECT().InChan().Times(numRecs).Return(recycleChan)
			delCall := ir.EXPECT().Deliver(pack).Times(numRecs)
			delCall.Do(func(pack *PipelinePack) {
				pack.Recycle()
			})

			for err == nil {
				err = sr.SplitStream(reader, nil)
			}
			c.Expect(err, gs.Equals, io.EOF)
		})

		c.Specify("correctly handles appends after EOF", func() {
			half := len(b) / 2
			reader := makeMockReader(b[:half])
			totalBytesRead := 0

			count, errCount, bytesRead, foundEOFCount, _, finalRecordLength,
				eofRecordLength := readRecordsFromStream(sr, reader, false)
			totalBytesRead += bytesRead

			c.Expect(errCount, gs.Equals, 0)
			c.Expect(count, gs.Equals, 25)
			c.Expect(foundEOFCount, gs.Equals, 25)
			c.Expect(finalRecordLength, gs.Equals, 215)
			c.Expect(eofRecordLength, gs.Equals, 0)
			c.Expect(bytesRead <= half, gs.IsTrue)

			reader.Append(b[half:])

			count, errCount, bytesRead, foundEOFCount,
				remainingDataLength, finalRecordLength,
				eofRecordLength := readRecordsFromStream(sr, reader, true)
			totalBytesRead += bytesRead
			c.Expect(errCount, gs.Equals, 0)
			c.Expect(count, gs.Equals, 25)
			c.Expect(foundEOFCount, gs.Equals, 25)
			c.Expect(remainingDataLength, gs.Equals, 0)
			c.Expect(finalRecordLength, gs.Equals, 215)
			c.Expect(eofRecordLength, gs.Equals, 0)

			c.Expect(totalBytesRead, gs.Equals, len(b))
		})

		c.Specify("reuse on another stream without GetRemainingData", func() {
			// Test the case where we reuse the same SplitterRunner on
			// two different readers, and we do not call GetRemainingData before
			// using the second reader.
			half := len(b) / 2
			reader1 := makeMockReader(b[:half])

			count, errCount, bytesRead, foundEOFCount, _, finalRecordLength,
				eofRecordLength := readRecordsFromStream(sr, reader1, false)

			c.Expect(errCount, gs.Equals, 0)
			c.Expect(count, gs.Equals, 25)
			c.Expect(foundEOFCount, gs.Equals, 25)
			c.Expect(finalRecordLength, gs.Equals, 215)
			c.Expect(eofRecordLength, gs.Equals, 0)

			leftovers := half - bytesRead
			c.Expect(leftovers > 0, gs.IsTrue)

			reader2 := makeMockReader(b)

			// Don't call GetRemainingData before using sr on a new stream
			count, errCount, bytesRead, foundEOFCount, remainingDataLength, finalRecordLength,
				eofRecordLength := readRecordsFromStream(sr, reader2, true)

			c.Expect(errCount, gs.Equals, 0)
			c.Expect(count, gs.Equals, 50)
			c.Expect(foundEOFCount, gs.Equals, 50)
			c.Expect(remainingDataLength, gs.Equals, 0)
			c.Expect(finalRecordLength, gs.Equals, 215)
			c.Expect(eofRecordLength, gs.Equals, 0)
			// sr misreports the "remaining data" piece from reader1 as being
			// read from reader2
			c.Expect(bytesRead, gs.Equals, len(b)+leftovers)
		})

		c.Specify("reuse on another stream with reset", func() {
			// Test the case where we reuse the same SplitterRunner on
			// two different readers, but we call GetRemainingData before using
			// the second reader.
			half := len(b) / 2
			reader1 := makeMockReader(b[:half])

			count, errCount, bytesRead, foundEOFCount, _, finalRecordLength,
				eofRecordLength := readRecordsFromStream(sr, reader1, false)

			c.Expect(errCount, gs.Equals, 0)
			c.Expect(count, gs.Equals, 25)
			c.Expect(foundEOFCount, gs.Equals, 25)
			c.Expect(finalRecordLength, gs.Equals, 215)
			c.Expect(eofRecordLength, gs.Equals, 0)

			leftovers := half - bytesRead
			c.Expect(leftovers > 0, gs.IsTrue)

			reader2 := makeMockReader(b)

			// Call GetRemainingData before using sr on a new stream
			sr.GetRemainingData()
			count, errCount, bytesRead, foundEOFCount, remainingDataLength, finalRecordLength,
				eofRecordLength := readRecordsFromStream(sr, reader2, true)

			c.Expect(errCount, gs.Equals, 0)
			c.Expect(count, gs.Equals, 50)
			c.Expect(foundEOFCount, gs.Equals, 50)
			c.Expect(remainingDataLength, gs.Equals, 0)
			c.Expect(finalRecordLength, gs.Equals, 215)
			c.Expect(eofRecordLength, gs.Equals, 0)
			// Now we see the correct number of bytes being read.
			c.Expect(bytesRead, gs.Equals, len(b))
		})
	})

	c.Specify("A SplitterRunner w/ TokenSplitter", func() {
		splitter := &TokenSplitter{}
		config := splitter.ConfigStruct().(*TokenSplitterConfig)

		c.Specify("sets readPos to 0 when read returns ErrShortBuffer", func() {
			config.Delimiter = "\t"
			err := splitter.Init(config)
			c.Assume(err, gs.IsNil)

			sr := NewSplitterRunner("TokenSplitter", splitter, srConfig)

			b := make([]byte, message.MAX_RECORD_SIZE+1)
			reader := bytes.NewReader(b)

			var n int
			var record []byte
			for err == nil {
				n, record, err = sr.GetRecordFromStream(reader)
			}
			c.Expect(n, gs.Equals, int(message.MAX_RECORD_SIZE))
			c.Expect(len(record), gs.Equals, 0)
			c.Expect(err, gs.Equals, io.ErrShortBuffer)
			c.Expect(sr.readPos, gs.Equals, 0)
			c.Expect(sr.scanPos, gs.Equals, 0)
		})

		c.Specify("checks if splitter honors 'deliver_incomplete_final' setting", func() {

			config.Count = 4
			numRecs := 10
			err := splitter.Init(config)
			c.Assume(err, gs.IsNil)

			packSupply := make(chan *PipelinePack, 1)
			pack := NewPipelinePack(packSupply)
			packSupply <- pack
			ir := NewMockInputRunner(ctrl)
			// ir.EXPECT().InChan().Return(packSupply).Times(numRecs)
			// ir.EXPECT().Name().Return("foo").Times(numRecs)
			ir.EXPECT().InChan().Return(packSupply).AnyTimes()
			ir.EXPECT().Name().Return("foo").AnyTimes()

			incompleteFinal := true
			srConfig.IncompleteFinal = &incompleteFinal
			sr := NewSplitterRunner("TokenSplitter", splitter, srConfig)
			sr.ir = ir

			rExpected := []byte("test1\ntest12\ntest123\npartial\n")
			buf := bytes.Repeat(rExpected, numRecs)
			buf = buf[:len(buf)-1] // 40 lines separated by 39 newlines

			reader := bytes.NewReader(buf)
			mockDel := NewMockDeliverer(ctrl)
			delCall := mockDel.EXPECT().Deliver(gomock.Any()).AnyTimes()
			i := 0
			delCall.Do(func(pack *PipelinePack) {
				i++
				if i < numRecs {
					c.Expect(pack.Message.GetPayload(), gs.Equals, string(rExpected))
				} else {
					c.Expect(pack.Message.GetPayload(), gs.Equals,
						string(rExpected[:len(rExpected)-1]))
				}
				pack.Recycle()
			})
			c.Specify("via SplitStream", func() {
				for err == nil {
					err = sr.SplitStream(reader, mockDel)
				}
				c.Expect(err, gs.Equals, io.EOF)
				c.Expect(i, gs.Equals, numRecs)
			})
			c.Specify("via SplitBytes", func() {
				seekPos, err := sr.SplitBytes(buf, mockDel)
				c.Assume(err, gs.IsNil)
				c.Expect(seekPos, gs.Equals, len(buf))
				c.Expect(i, gs.Equals, numRecs)
			})
		})

	})
}
Esempio n. 12
0
func AMQPPluginSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)

	// Our two user/conn waitgroups.
	ug := new(sync.WaitGroup)
	cg := new(sync.WaitGroup)

	// Setup the mock channel.
	mch := NewMockAMQPChannel(ctrl)

	// Setup the mock amqpHub with the mock chan return.
	aqh := NewMockAMQPConnectionHub(ctrl)
	aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)

	errChan := make(chan error, 1)
	bytesChan := make(chan []byte, 1)

	c.Specify("An amqp input", func() {
		// Setup all the mock calls for Init.
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)
		mch.EXPECT().QueueDeclare("", false, true, false, false,
			gomock.Any()).Return(amqp.Queue{}, nil)
		mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
		mch.EXPECT().Qos(2, 0, false).Return(nil)

		ith := new(plugins_ts.InputTestHelper)
		ith.Msg = pipeline_ts.GetTestMessage()
		ith.Pack = NewPipelinePack(config.InputRecycleChan())

		// Set up relevant mocks.
		ith.MockHelper = NewMockPluginHelper(ctrl)
		ith.MockInputRunner = NewMockInputRunner(ctrl)
		ith.MockSplitterRunner = NewMockSplitterRunner(ctrl)
		ith.PackSupply = make(chan *PipelinePack, 1)

		ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)

		amqpInput := new(AMQPInput)
		amqpInput.amqpHub = aqh
		config := amqpInput.ConfigStruct().(*AMQPInputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"
		config.QueueTTL = 300000

		err := amqpInput.Init(config)
		c.Assume(err, gs.IsNil)
		c.Expect(amqpInput.ch, gs.Equals, mch)

		c.Specify("consumes a text message", func() {
			// Create a channel to send data to the input. Drop a message on
			// there and close the channel.
			streamChan := make(chan amqp.Delivery, 1)
			ack := plugins_ts.NewMockAcknowledger(ctrl)
			ack.EXPECT().Ack(gomock.Any(), false)
			streamChan <- amqp.Delivery{
				ContentType:  "text/plain",
				Body:         []byte("This is a message"),
				Timestamp:    time.Now(),
				Acknowledger: ack,
			}
			mch.EXPECT().Consume("", "", false, false, false, false,
				gomock.Any()).Return(streamChan, nil)

			// Increase the usage since Run decrements it on close.
			ug.Add(1)

			splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
				nil)
			splitCall.Do(func(recd []byte, del Deliverer) {
				bytesChan <- recd
			})
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			go func() {
				err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
				errChan <- err
			}()

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "This is a message")
			close(streamChan)
			err = <-errChan
		})

		c.Specify("consumes a protobuf encoded message", func() {
			encoder := client.NewProtobufEncoder(nil)
			streamChan := make(chan amqp.Delivery, 1)

			msg := new(message.Message)
			msg.SetUuid(uuid.NewRandom())
			msg.SetTimestamp(time.Now().UnixNano())
			msg.SetType("logfile")
			msg.SetLogger("/a/nice/path")
			msg.SetSeverity(int32(0))
			msg.SetEnvVersion("0.2")
			msg.SetPid(0)
			msg.SetPayload("This is a message")
			msg.SetHostname("TestHost")

			msgBody := make([]byte, 0, 500)
			_ = encoder.EncodeMessageStream(msg, &msgBody)

			ack := plugins_ts.NewMockAcknowledger(ctrl)
			ack.EXPECT().Ack(gomock.Any(), false)

			streamChan <- amqp.Delivery{
				ContentType:  "application/hekad",
				Body:         msgBody,
				Timestamp:    time.Now(),
				Acknowledger: ack,
			}
			mch.EXPECT().Consume("", "", false, false, false, false,
				gomock.Any()).Return(streamChan, nil)

			// Increase the usage since Run decrements it on close.
			ug.Add(1)

			splitCall := ith.MockSplitterRunner.EXPECT().SplitBytes(gomock.Any(),
				nil)
			splitCall.Do(func(recd []byte, del Deliverer) {
				bytesChan <- recd
			})
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(true)
			go func() {
				err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
				errChan <- err
			}()

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, string(msgBody))
			close(streamChan)
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})
	})

	c.Specify("An amqp output", func() {
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		pConfig := NewPipelineConfig(nil)

		amqpOutput := new(AMQPOutput)
		amqpOutput.amqpHub = aqh
		config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"

		closeChan := make(chan *amqp.Error)

		inChan := make(chan *PipelinePack, 1)

		mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)

		// Increase the usage since Run decrements it on close.
		ug.Add(1)

		// Expect the close and the InChan calls.
		aqh.EXPECT().Close("", cg)
		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("publishes a plain message", func() {
			encoder := new(plugins.PayloadEncoder)
			econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
			econfig.AppendNewlines = false
			encoder.Init(econfig)
			payloadBytes, err := encoder.Encode(pack)

			config.Encoder = "PayloadEncoder"
			config.ContentType = "text/plain"
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()

			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})

		c.Specify("publishes a serialized message", func() {
			encoder := new(ProtobufEncoder)
			encoder.SetPipelineConfig(pConfig)
			encoder.Init(nil)
			protoBytes, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()
			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})
	})
}
Esempio n. 13
0
func TestSocketOrigin(t *testing.T) {
	var err error

	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()
	mckStat := NewMockStatistician(mockCtrl)
	mckStore := NewMockStore(mockCtrl)
	mckRouter := NewMockRouter(mockCtrl)

	app := NewApplication()
	app.SetLogger(mckLogger)
	app.SetMetrics(mckStat)
	app.SetStore(mckStore)
	app.SetRouter(mckRouter)

	sh := NewSocketHandler()
	defer sh.Close()
	sh.setApp(app)

	pipe := newPipeListener()
	defer pipe.Close()
	if err := sh.listenWithConfig(listenerConfig{listener: pipe}); err != nil {
		t.Fatalf("Error setting listener: %s", err)
	}
	sh.server = newServeWaiter(&http.Server{Handler: sh.ServeMux()})
	app.SetSocketHandler(sh)

	errChan := make(chan error, 1)
	go sh.Start(errChan)

	uaid := "5e1e5984569c4f00bf4bea47754a6403"
	gomock.InOrder(
		mckStat.EXPECT().Increment("client.socket.connect"),
		mckStore.EXPECT().CanStore(0).Return(true),
		mckRouter.EXPECT().Register(uaid),
		mckStat.EXPECT().Increment("updates.client.hello"),
		mckStore.EXPECT().FetchAll(uaid, gomock.Any()).Return(nil, nil, nil),
		mckStat.EXPECT().Timer("client.flush", gomock.Any()),
		mckRouter.EXPECT().Unregister(uaid),
		mckStat.EXPECT().Timer("client.socket.lifespan", gomock.Any()),
		mckStat.EXPECT().Increment("client.socket.disconnect"),
	)

	origin := &url.URL{Scheme: "https", Host: "example.com"}
	conn, err := dialSocketListener(pipe, &websocket.Config{
		Location: origin,
		Origin:   origin,
		Version:  websocket.ProtocolVersionHybi13,
	})
	if err != nil {
		t.Fatalf("Error dialing origin: %s", err)
	}
	defer conn.Close()
	err = websocket.JSON.Send(conn, struct {
		Type       string   `json:"messageType"`
		DeviceID   string   `json:"uaid"`
		ChannelIDs []string `json:"channelIDs"`
	}{"hello", uaid, []string{}})
	if err != nil {
		t.Fatalf("Error writing client handshake: %s", err)
	}
	reply := new(HelloReply)
	if err = websocket.JSON.Receive(conn, reply); err != nil {
		t.Fatalf("Error reading server handshake: %s", err)
	}
	if reply.DeviceID != uaid {
		t.Fatalf("Mismatched device ID: got %q; want %q", reply.DeviceID, uaid)
	}

	worker, workerConnected := app.GetWorker(uaid)
	if !workerConnected {
		t.Fatalf("Missing worker for device ID %q", uaid)
	}
	workerOrigin := worker.Origin()
	if expectedOrigin := origin.String(); workerOrigin != expectedOrigin {
		t.Errorf("Mismatched origins: got %q; want %q",
			workerOrigin, expectedOrigin)
	}
}
Esempio n. 14
0
func HttpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)

	json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`

	c.Specify("A HttpInput", func() {

		httpInput := HttpInput{}
		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)

		runOutputChan := make(chan string, 1)
		startInput := func() {
			go func() {
				err := httpInput.Run(ith.MockInputRunner, ith.MockHelper)
				var runOutput string
				if err != nil {
					runOutput = err.Error()
				}
				runOutputChan <- runOutput
			}()
		}

		ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
		ith.PackSupply = make(chan *PipelinePack, 1)
		ith.PackSupply <- ith.Pack

		config := httpInput.ConfigStruct().(*HttpInputConfig)

		c.Specify("short circuits packs into the router", func() {
			config.Url = "http://localhost:9876/"
			tickChan := make(chan time.Time)

			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)

			ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
			ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

			err := httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()

			tickChan <- time.Now()

			// We need for the pipeline to finish up
			time.Sleep(50 * time.Millisecond)
		})

		c.Specify("with a decoder", func() {

			decoderName := "TestDecoder"
			config.DecoderName = decoderName

			c.Specify("honors time ticker to flush", func() {
				// Spin up a http server
				server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876)
				c.Expect(err, gs.IsNil)
				go server.Start("/")
				time.Sleep(10 * time.Millisecond)

				config.Url = "http://localhost:9876/"
				tickChan := make(chan time.Time)

				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)

				ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
				ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

				mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

				// Stub out the DecoderRunner input channel so that we can
				// inspect bytes later on
				dRunnerInChan := make(chan *PipelinePack, 1)
				mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)

				ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
				ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)

				err = httpInput.Init(config)
				c.Assume(err, gs.IsNil)
				startInput()

				tickChan <- time.Now()

				// We need for the pipeline to finish up
				time.Sleep(50 * time.Millisecond)
			})

			c.Specify("supports configuring HTTP Basic Authentication", func() {
				// Spin up a http server which expects username "user" and password "password"
				server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875)
				c.Expect(err, gs.IsNil)
				go server.Start("/BasicAuthTest")
				time.Sleep(10 * time.Millisecond)

				config.Url = "http://localhost:9875/BasicAuthTest"
				config.User = "******"
				config.Password = "******"
				tickChan := make(chan time.Time)

				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)

				ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
				ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

				mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

				// Stub out the DecoderRunner input channel so that we can
				// inspect bytes later on
				dRunnerInChan := make(chan *PipelinePack, 1)
				mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)

				ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
				ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)

				err = httpInput.Init(config)
				c.Assume(err, gs.IsNil)
				startInput()

				tickChan <- time.Now()

				// we expect a statuscode 200 (i.e. success)
				pack := <-dRunnerInChan
				statusCode, ok := pack.Message.GetFieldValue("StatusCode")
				c.Assume(ok, gs.IsTrue)
				c.Expect(statusCode, gs.Equals, int64(200))

				// We need for the pipeline to finish up
				time.Sleep(50 * time.Millisecond)
			})

			c.Specify("supports configuring a different HTTP method", func() {
				// Spin up a http server which expects requests with method "POST"
				server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874)
				c.Expect(err, gs.IsNil)
				go server.Start("/PostTest")
				time.Sleep(10 * time.Millisecond)

				config.Url = "http://localhost:9874/PostTest"
				config.Method = "POST"
				tickChan := make(chan time.Time)

				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)

				ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
				ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

				mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

				// Stub out the DecoderRunner input channel so that we can
				// inspect bytes later on
				dRunnerInChan := make(chan *PipelinePack, 1)
				mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)

				ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
				ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)

				err = httpInput.Init(config)
				c.Assume(err, gs.IsNil)
				startInput()

				tickChan <- time.Now()

				// we expect a statuscode 200 (i.e. success)
				pack := <-dRunnerInChan
				statusCode, ok := pack.Message.GetFieldValue("StatusCode")
				c.Assume(ok, gs.IsTrue)
				c.Expect(statusCode, gs.Equals, int64(200))

				// We need for the pipeline to finish up
				time.Sleep(50 * time.Millisecond)
			})

			c.Specify("supports configuring HTTP headers", func() {
				// Spin up a http server which expects requests with method "POST"
				server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873)
				c.Expect(err, gs.IsNil)
				go server.Start("/HeadersTest")
				time.Sleep(10 * time.Millisecond)

				config.Url = "http://localhost:9873/HeadersTest"
				config.Headers = map[string]string{"Accept": "text/plain"}

				tickChan := make(chan time.Time)

				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)

				ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
				ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

				mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

				// Stub out the DecoderRunner input channel so that we can
				// inspect bytes later on
				dRunnerInChan := make(chan *PipelinePack, 1)
				mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)

				ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
				ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)

				err = httpInput.Init(config)
				c.Assume(err, gs.IsNil)
				startInput()

				tickChan <- time.Now()

				// we expect a statuscode 200 (i.e. success)
				pack := <-dRunnerInChan
				statusCode, ok := pack.Message.GetFieldValue("StatusCode")
				c.Assume(ok, gs.IsTrue)
				c.Expect(statusCode, gs.Equals, int64(200))

				// We need for the pipeline to finish up
				time.Sleep(50 * time.Millisecond)
			})

			c.Specify("supports configuring a request body", func() {
				// Spin up a http server that echoes back the request body
				server, err := plugins_ts.NewHttpBodyServer("localhost", 9872)
				c.Expect(err, gs.IsNil)
				go server.Start("/BodyTest")
				time.Sleep(10 * time.Millisecond)

				config.Url = "http://localhost:9872/BodyTest"
				config.Method = "POST"
				config.Body = json_post

				tickChan := make(chan time.Time)

				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)

				ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
				ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

				mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

				// Stub out the DecoderRunner input channel so that we can
				// inspect bytes later on
				dRunnerInChan := make(chan *PipelinePack, 1)
				mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)

				ith.MockInputRunner.EXPECT().Name().Return("HttpInput")
				ith.MockHelper.EXPECT().DecoderRunner(decoderName, "HttpInput-TestDecoder").Return(mockDecoderRunner, true)

				err = httpInput.Init(config)
				c.Assume(err, gs.IsNil)
				startInput()

				tickChan <- time.Now()

				pack := <-dRunnerInChan
				c.Expect(*pack.Message.Payload, gs.Equals, json_post)

				// We need for the pipeline to finish up
				time.Sleep(50 * time.Millisecond)
			})

			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any())
			httpInput.Stop()
			runOutput := <-runOutputChan
			c.Expect(runOutput, gs.Equals, "")
		})
	})
}
Esempio n. 15
0
func ProcessInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()

	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	c.Specify("A ProcessInput", func() {
		pInput := ProcessInput{}

		config := pInput.ConfigStruct().(*ProcessInputConfig)
		config.Command = make(map[string]cmdConfig)

		ith.MockHelper.EXPECT().Hostname().Return(pConfig.Hostname())

		tickChan := make(chan time.Time)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

		errChan := make(chan error)

		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)

		decChan := make(chan func(*PipelinePack), 1)
		setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
		setDecCall.Do(func(dec func(*PipelinePack)) {
			decChan <- dec
		})

		bytesChan := make(chan []byte, 1)
		splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
			ith.MockDeliverer).Return(nil)
		splitCall.Do(func(r io.Reader, del Deliverer) {
			bytes, err := ioutil.ReadAll(r)
			c.Assume(err, gs.IsNil)
			bytesChan <- bytes
		})

		ith.MockDeliverer.EXPECT().Done()

		c.Specify("using stdout", func() {
			ith.MockInputRunner.EXPECT().NewDeliverer("stdout").Return(ith.MockDeliverer)
			ith.MockInputRunner.EXPECT().NewSplitterRunner("stdout").Return(
				ith.MockSplitterRunner)

			c.Specify("reads a message from ProcessInput", func() {
				pInput.SetName("SimpleTest")

				// Note that no working directory is explicitly specified.
				config.Command["0"] = cmdConfig{
					Bin:  PROCESSINPUT_TEST1_CMD,
					Args: PROCESSINPUT_TEST1_CMD_ARGS,
				}
				err := pInput.Init(config)
				c.Assume(err, gs.IsNil)

				go func() {
					errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
				}()
				tickChan <- time.Now()

				actual := <-bytesChan
				c.Expect(string(actual), gs.Equals, PROCESSINPUT_TEST1_OUTPUT+"\n")

				dec := <-decChan
				dec(ith.Pack)
				fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName")
				c.Expect(fPInputName.ValueString[0], gs.Equals, "SimpleTest.stdout")

				pInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})

			c.Specify("can pipe multiple commands together", func() {
				pInput.SetName("PipedCmd")

				// Note that no working directory is explicitly specified.
				config.Command["0"] = cmdConfig{
					Bin:  PROCESSINPUT_PIPE_CMD1,
					Args: PROCESSINPUT_PIPE_CMD1_ARGS,
				}
				config.Command["1"] = cmdConfig{
					Bin:  PROCESSINPUT_PIPE_CMD2,
					Args: PROCESSINPUT_PIPE_CMD2_ARGS,
				}
				err := pInput.Init(config)
				c.Assume(err, gs.IsNil)

				go func() {
					errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
				}()
				tickChan <- time.Now()

				actual := <-bytesChan
				c.Expect(string(actual), gs.Equals, PROCESSINPUT_PIPE_OUTPUT+"\n")

				dec := <-decChan
				dec(ith.Pack)
				fPInputName := ith.Pack.Message.FindFirstField("ProcessInputName")
				c.Expect(fPInputName.ValueString[0], gs.Equals, "PipedCmd.stdout")

				pInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("using stderr", func() {
			ith.MockInputRunner.EXPECT().NewDeliverer("stderr").Return(ith.MockDeliverer)
			ith.MockInputRunner.EXPECT().NewSplitterRunner("stderr").Return(
				ith.MockSplitterRunner)

			c.Specify("handles bad arguments", func() {
				pInput.SetName("BadArgs")
				config.ParseStdout = false
				config.ParseStderr = true

				// Note that no working directory is explicitly specified.
				config.Command["0"] = cmdConfig{Bin: STDERR_CMD, Args: STDERR_CMD_ARGS}

				err := pInput.Init(config)
				c.Assume(err, gs.IsNil)

				expectedErr := fmt.Errorf(
					"BadArgs CommandChain::Wait() error: [Subcommand returned an error: [exit status 1]]")
				ith.MockInputRunner.EXPECT().LogError(expectedErr)

				go func() {
					errChan <- pInput.Run(ith.MockInputRunner, ith.MockHelper)
				}()
				tickChan <- time.Now()

				// Error message differs by platform, but we at least wait
				// until we get it.
				<-bytesChan

				pInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})
	})
}
Esempio n. 16
0
func HttpListenInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	httpListenInput := HttpListenInput{}
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	splitter := &TokenSplitter{} // Not actually used.

	errChan := make(chan error, 1)
	startInput := func() {
		go func() {
			err := httpListenInput.Run(ith.MockInputRunner, ith.MockHelper)
			errChan <- err
		}()
	}

	config := httpListenInput.ConfigStruct().(*HttpListenInputConfig)
	config.Address = "127.0.0.1:58325"

	c.Specify("A HttpListenInput", func() {
		startedChan := make(chan bool, 1)
		defer close(startedChan)
		ts := httptest.NewUnstartedServer(nil)

		httpListenInput.starterFunc = func(hli *HttpListenInput) error {
			ts.Start()
			startedChan <- true
			return nil
		}

		// These EXPECTs imply that every spec below will send exactly one
		// HTTP request to the input.
		ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
			ith.MockSplitterRunner)
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)

		decChan := make(chan func(*PipelinePack), 1)
		feedDecorator := func(decorator func(*PipelinePack)) {
			decChan <- decorator
		}
		setDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
		setDecCall.Do(feedDecorator)

		streamChan := make(chan io.Reader, 1)
		feedStream := func(r io.Reader) {
			streamChan <- r
		}
		getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
			gomock.Any()).Do(feedStream)

		bytesChan := make(chan []byte, 1)
		deliver := func(msgBytes []byte, del Deliverer) {
			bytesChan <- msgBytes
		}

		ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()

		c.Specify("Adds query parameters to the message pack as fields", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			getRecCall.Return(0, make([]byte, 0), io.EOF)
			startInput()
			<-startedChan
			resp, err := http.Get(ts.URL + "/?test=Hello%20World")
			resp.Body.Close()
			c.Assume(err, gs.IsNil)
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			fieldValue, ok := ith.Pack.Message.GetFieldValue("test")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "Hello World")
		})

		c.Specify("Add custom headers", func() {
			config.Headers = http.Header{
				"One":  []string{"two", "three"},
				"Four": []string{"five", "six", "seven"},
			}
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			getRecCall.Return(0, make([]byte, 0), io.EOF)
			startInput()
			<-startedChan
			resp, err := http.Get(ts.URL)
			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			// Verify headers are there
			eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
			c.Expect(eq, gs.IsTrue)
			eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
			c.Expect(eq, gs.IsTrue)
		})

		c.Specify("Request body is sent as record", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			body := "1+2"
			getRecCall.Return(0, []byte(body), io.EOF)
			startInput()
			<-startedChan
			deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
				nil)
			deliverCall.Do(deliver)
			resp, err := http.Post(ts.URL, "text/plain", strings.NewReader(body))
			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "1+2")
		})

		c.Specify("Add request headers as fields", func() {
			config.RequestHeaders = []string{
				"X-REQUEST-ID",
			}
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			getRecCall.Return(0, make([]byte, 0), io.EOF)
			startInput()
			<-startedChan

			client := &http.Client{}
			req, err := http.NewRequest("GET", ts.URL, nil)
			req.Header.Add("X-REQUEST-ID", "12345")
			resp, err := client.Do(req)

			c.Assume(err, gs.IsNil)
			resp.Body.Close()
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)
			fieldValue, ok := ith.Pack.Message.GetFieldValue("X-REQUEST-ID")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "12345")
		})

		c.Specify("Add other request properties as fields", func() {
			err := httpListenInput.Init(config)
			c.Assume(err, gs.IsNil)
			ts.Config = httpListenInput.server

			getRecCall.Return(0, make([]byte, 0), io.EOF)
			startInput()
			<-startedChan

			client := &http.Client{}
			req, err := http.NewRequest("GET", ts.URL+"/foo/bar?baz=2", nil)
			c.Assume(err, gs.IsNil)
			req.Host = "incoming.example.com:8080"
			resp, err := client.Do(req)
			resp.Body.Close()
			c.Assume(err, gs.IsNil)
			c.Assume(resp.StatusCode, gs.Equals, 200)

			packDec := <-decChan
			packDec(ith.Pack)

			fieldValue, ok := ith.Pack.Message.GetFieldValue("Host")
			c.Assume(ok, gs.IsTrue)
			// Host should not include port number.
			c.Expect(fieldValue, gs.Equals, "incoming.example.com")
			fieldValue, ok = ith.Pack.Message.GetFieldValue("Path")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue, gs.Equals, "/foo/bar")

			hostname, err := os.Hostname()
			c.Assume(err, gs.IsNil)
			c.Expect(*ith.Pack.Message.Hostname, gs.Equals, hostname)
			c.Expect(*ith.Pack.Message.EnvVersion, gs.Equals, "1")

			fieldValue, ok = ith.Pack.Message.GetFieldValue("RemoteAddr")
			c.Assume(ok, gs.IsTrue)
			c.Expect(fieldValue != nil, gs.IsTrue)
			fieldValueStr, ok := fieldValue.(string)
			c.Assume(ok, gs.IsTrue)
			c.Expect(len(fieldValueStr) > 0, gs.IsTrue)
			// Per the `Request` docs, this should be an IP address:
			// http://golang.org/pkg/net/http/#Request
			ip := net.ParseIP(fieldValueStr)
			c.Expect(ip != nil, gs.IsTrue)
		})

		ts.Close()
		httpListenInput.Stop()
		err := <-errChan
		c.Expect(err, gs.IsNil)
	})
}
Esempio n. 17
0
func AMQPPluginSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	config := NewPipelineConfig(nil)

	// Our two user/conn waitgroups
	ug := new(sync.WaitGroup)
	cg := new(sync.WaitGroup)

	// Setup the mock channel
	mch := NewMockAMQPChannel(ctrl)

	// Setup the mock amqpHub with the mock chan return
	aqh := NewMockAMQPConnectionHub(ctrl)
	aqh.EXPECT().GetChannel("", AMQPDialer{}).Return(mch, ug, cg, nil)

	errChan := make(chan error, 1)

	c.Specify("An amqp input", func() {
		// Setup all the mock calls for Init
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)
		mch.EXPECT().QueueDeclare("", false, true, false, false,
			gomock.Any()).Return(amqp.Queue{}, nil)
		mch.EXPECT().QueueBind("", "test", "", false, gomock.Any()).Return(nil)
		mch.EXPECT().Qos(2, 0, false).Return(nil)

		ith := new(plugins_ts.InputTestHelper)
		ith.Msg = pipeline_ts.GetTestMessage()
		ith.Pack = NewPipelinePack(config.InputRecycleChan())

		// set up mock helper, decoder set, and packSupply channel
		ith.MockHelper = NewMockPluginHelper(ctrl)
		ith.MockInputRunner = NewMockInputRunner(ctrl)
		mockDRunner := NewMockDecoderRunner(ctrl)
		ith.PackSupply = make(chan *PipelinePack, 1)
		ith.DecodeChan = make(chan *PipelinePack)

		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)

		amqpInput := new(AMQPInput)
		amqpInput.amqpHub = aqh
		config := amqpInput.ConfigStruct().(*AMQPInputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"
		config.QueueTTL = 300000

		c.Specify("with a valid setup and no decoder", func() {
			err := amqpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpInput.ch, gs.Equals, mch)

			c.Specify("consumes a message", func() {

				// Create a channel to send data to the input
				// Drop a message on there and close the channel
				streamChan := make(chan amqp.Delivery, 1)
				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)
				streamChan <- amqp.Delivery{
					ContentType:  "text/plain",
					Body:         []byte("This is a message"),
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the injected packet
				ith.MockInputRunner.EXPECT().Inject(gomock.Any())

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
				c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
				c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
			})
		})

		c.Specify("with a valid setup using a decoder", func() {
			decoderName := "defaultDecoder"
			config.Decoder = decoderName
			err := amqpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpInput.ch, gs.Equals, mch)

			// Mock up our default decoder runner and decoder.
			ith.MockInputRunner.EXPECT().Name().Return("AMQPInput")
			decCall := ith.MockHelper.EXPECT().DecoderRunner(decoderName, "AMQPInput-defaultDecoder")
			decCall.Return(mockDRunner, true)
			mockDecoder := NewMockDecoder(ctrl)
			mockDRunner.EXPECT().Decoder().Return(mockDecoder)

			c.Specify("consumes a message", func() {
				packs := []*PipelinePack{ith.Pack}
				mockDecoder.EXPECT().Decode(ith.Pack).Return(packs, nil)

				// Create a channel to send data to the input
				// Drop a message on there and close the channel
				streamChan := make(chan amqp.Delivery, 1)
				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)
				streamChan <- amqp.Delivery{
					ContentType:  "text/plain",
					Body:         []byte("This is a message"),
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the injected packet
				ith.MockInputRunner.EXPECT().Inject(gomock.Any())

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(ith.Pack.Message.GetType(), gs.Equals, "amqp")
				c.Expect(ith.Pack.Message.GetPayload(), gs.Equals, "This is a message")
			})

			c.Specify("consumes a serialized message", func() {
				encoder := client.NewProtobufEncoder(nil)
				streamChan := make(chan amqp.Delivery, 1)

				msg := new(message.Message)
				msg.SetUuid(uuid.NewRandom())
				msg.SetTimestamp(time.Now().UnixNano())
				msg.SetType("logfile")
				msg.SetLogger("/a/nice/path")
				msg.SetSeverity(int32(0))
				msg.SetEnvVersion("0.2")
				msg.SetPid(0)
				msg.SetPayload("This is a message")
				msg.SetHostname("TestHost")

				msgBody := make([]byte, 0, 500)
				_ = encoder.EncodeMessageStream(msg, &msgBody)

				ack := plugins_ts.NewMockAcknowledger(ctrl)
				ack.EXPECT().Ack(gomock.Any(), false)

				streamChan <- amqp.Delivery{
					ContentType:  "application/hekad",
					Body:         msgBody,
					Timestamp:    time.Now(),
					Acknowledger: ack,
				}
				mch.EXPECT().Consume("", "", false, false, false, false,
					gomock.Any()).Return(streamChan, nil)

				// Expect the decoded pack
				mockDRunner.EXPECT().InChan().Return(ith.DecodeChan)

				// Increase the usage since Run decrements it on close
				ug.Add(1)

				ith.PackSupply <- ith.Pack
				go func() {
					err := amqpInput.Run(ith.MockInputRunner, ith.MockHelper)
					errChan <- err
				}()
				packRef := <-ith.DecodeChan
				c.Expect(ith.Pack, gs.Equals, packRef)
				// Ignore leading 5 bytes of encoded message as thats the header
				c.Expect(string(packRef.MsgBytes), gs.Equals, string(msgBody[5:]))
				ith.PackSupply <- ith.Pack
				close(streamChan)
				err = <-errChan
				c.Expect(err, gs.IsNil)
			})
		})
	})

	c.Specify("An amqp output", func() {
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		pConfig := NewPipelineConfig(nil)

		amqpOutput := new(AMQPOutput)
		amqpOutput.amqpHub = aqh
		config := amqpOutput.ConfigStruct().(*AMQPOutputConfig)
		config.URL = ""
		config.Exchange = ""
		config.ExchangeType = ""
		config.RoutingKey = "test"

		closeChan := make(chan *amqp.Error)

		inChan := make(chan *PipelinePack, 1)

		mch.EXPECT().NotifyClose(gomock.Any()).Return(closeChan)
		mch.EXPECT().ExchangeDeclare("", "", false, true, false, false,
			gomock.Any()).Return(nil)

		// Increase the usage since Run decrements it on close
		ug.Add(1)

		// Expect the close and the InChan calls
		aqh.EXPECT().Close("", cg)
		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("publishes a plain message", func() {
			encoder := new(plugins.PayloadEncoder)
			econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
			econfig.AppendNewlines = false
			encoder.Init(econfig)
			payloadBytes, err := encoder.Encode(pack)

			config.Encoder = "PayloadEncoder"
			config.ContentType = "text/plain"
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(payloadBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()

			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})

		c.Specify("publishes a serialized message", func() {
			encoder := new(ProtobufEncoder)
			encoder.SetPipelineConfig(pConfig)
			encoder.Init(nil)
			protoBytes, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(protoBytes, nil)

			err = amqpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(amqpOutput.ch, gs.Equals, mch)

			mch.EXPECT().Publish("", "test", false, false, gomock.Any()).Return(nil)
			inChan <- pack
			close(inChan)
			close(closeChan)

			go func() {
				err := amqpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				errChan <- err
			}()
			ug.Wait()
			err = <-errChan
			c.Expect(err, gs.IsNil)
		})
	})
}
Esempio n. 18
0
func TestEndpointInvalidParams(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()

	mckStat := NewMockStatistician(mockCtrl)

	Convey("Invalid update parameters", t, func() {
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)

		eh := NewEndpointHandler()
		eh.setApp(app)
		eh.setMaxDataLen(512)
		app.SetEndpointHandler(eh)

		Convey("Should require PUT requests", func() {
			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "POST",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
			}
			mckStat.EXPECT().Increment("updates.appserver.invalid")
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 405)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Method Not Allowed"`)
		})

		Convey("Should reject negative versions", func() {
			vals := make(url.Values)
			vals.Set("version", "-1")

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}
			mckStat.EXPECT().Increment("updates.appserver.invalid")
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 400)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Invalid Version"`)
		})

		Convey("Should reject invalid versions", func() {
			vals := make(url.Values)
			vals.Set("version", "abc")

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}
			mckStat.EXPECT().Increment("updates.appserver.invalid")
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 400)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Invalid Version"`)
		})

		Convey("Should reject oversized payloads", func() {
			vals := make(url.Values)
			vals.Set("data", randomText(eh.maxDataLen+1))

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}

			mckStat.EXPECT().Increment("updates.appserver.toolong")
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 413)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Data exceeds max length of 512 bytes"`)
		})
	})
}
Esempio n. 19
0
func Test_GCMSend(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(),
		gomock.Any()).AnyTimes()
	mckStat := &TestMetrics{}
	mckStat.Init(nil, nil)
	mckStore := NewMockStore(mockCtrl)
	mckEndHandler := NewMockHandler(mockCtrl)

	mckGCMClient := &mockGCMClient{t: t}
	Convey("GCM Proprietary Ping", t, func() {
		uaid := "deadbeef00000000000000000000"
		vers := int64(1)
		data := "i'm a little teapot"
		fakeConnect := []byte(`{"regid":"testing"}`)
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)
		app.SetStore(mckStore)
		app.SetEndpointHandler(mckEndHandler)

		mckGCMClient.reset()
		headers := make(http.Header, 1)
		headers.Set("Retry-After", "1")
		mckGCMClient.reply = &http.Response{
			Body:       respBody("Ok"),
			StatusCode: 503,
			Header:     headers,
		}

		mckStore.EXPECT().FetchPing(uaid).Return(fakeConnect, nil)

		testGcm := &GCMPing{
			logger:  app.Logger(),
			metrics: mckStat,
			store:   mckStore,
			apiKey:  "test_api_key",
		}
		conf := testGcm.ConfigStruct()
		err := testGcm.Init(app, conf)
		So(err, ShouldBeNil)
		testGcm.ReplaceClient(mckGCMClient)

		// For this test, we first respond with a fake 503 message that
		// requests a "retry after", we follow up with a second message
		// that returns success.
		//
		// Not sure why but when using gomock, calls to Increment are
		// not being registered, so falling back to older testMetrics
		// object.
		ok, err := testGcm.Send(uaid, vers, data)
		So(err, ShouldBeNil)
		So(ok, ShouldEqual, true)
		So(mckStat.Counters["ping.gcm.retry"], ShouldEqual, 1)
		So(mckStat.Counters["ping.gcm.success"], ShouldEqual, 1)
	})
}
Esempio n. 20
0
func TestEndpointDelivery(t *testing.T) {
	useMockFuncs()
	defer useStdFuncs()

	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()
	mckStat := NewMockStatistician(mockCtrl)
	mckStore := NewMockStore(mockCtrl)
	mckRouter := NewMockRouter(mockCtrl)
	mckWorker := NewMockWorker(mockCtrl)

	Convey("Update delivery", t, func() {
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)
		app.SetStore(mckStore)
		app.SetRouter(mckRouter)

		Convey("Should attempt local delivery if `AlwaysRoute` is disabled", func() {
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			Convey("Should route updates if the device is not connected", func() {
				uaid := "f7e9fc483f7344c398701b6fa0e85e4f"
				chid := "737b7a0d25674be4bb184f015fce02cf"
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, int64(3), timeNow().UTC(),
						"", "").Return(true, nil),
					mckStat.EXPECT().Increment("router.broadcast.hit"),
					mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
					mckStat.EXPECT().Increment("updates.appserver.received"),
				)
				ok := eh.deliver(nil, uaid, chid, 3, "", "")
				So(ok, ShouldBeTrue)
			})

			// A routing failure still stores the alert for potential
			// client delivery. We should only return 404 for absolute
			// failures (where the endpoint is no longer valid
			Convey("Should return a 202 if routing fails", func() {
				resp := httptest.NewRecorder()
				req := &http.Request{
					Method: "PUT",
					Header: http.Header{HeaderID: {"reqID"}},
					URL:    &url.URL{Path: "/update/123"},
					Body:   formReader(url.Values{"version": {"1"}}),
				}
				gomock.InOrder(
					mckStore.EXPECT().KeyToIDs("123").Return("123", "456", nil),
					mckStat.EXPECT().Increment("updates.appserver.incoming"),
					mckStore.EXPECT().Update("123", "456", int64(1)).Return(nil),
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, "123", "456", int64(1),
						gomock.Any(), "reqID", "").Return(false, nil),
					mckStat.EXPECT().Increment("router.broadcast.miss"),
					mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
					mckStat.EXPECT().Increment("updates.appserver.rejected"),
				)
				eh.ServeMux().ServeHTTP(resp, req)

				So(resp.Code, ShouldEqual, 202)
				body, isJSON := getJSON(resp.HeaderMap, resp.Body)
				So(isJSON, ShouldBeTrue)
				So(body.String(), ShouldEqual, "{}")
			})

			Convey("Should return a 404 if local delivery fails", func() {
				uaid := "9e98d6415d8e4fd099ab1bad7178f750"
				chid := "0eecf572e99f4d508666d8da6c0b15a9"
				app.AddWorker(uaid, mckWorker)

				gomock.InOrder(
					mckWorker.EXPECT().Send(chid, int64(3), "").Return(
						errors.New("client gone")),
					mckStat.EXPECT().Increment("updates.appserver.rejected"),
				)
				ok := eh.deliver(nil, uaid, chid, int64(3), "", "")
				So(ok, ShouldBeFalse)
			})

			Convey("Should return an error if storage is unavailable", func() {
				resp := httptest.NewRecorder()
				req := &http.Request{
					Method: "PUT",
					Header: http.Header{},
					URL:    &url.URL{Path: "/update/123"},
					Body:   formReader(url.Values{"version": {"2"}}),
				}
				updateErr := ErrInvalidChannel
				gomock.InOrder(
					mckStore.EXPECT().KeyToIDs("123").Return("123", "456", nil),
					mckStat.EXPECT().Increment("updates.appserver.incoming"),
					mckStore.EXPECT().Update("123", "456", int64(2)).Return(updateErr),
					mckStat.EXPECT().Increment("updates.appserver.error"),
				)
				eh.ServeMux().ServeHTTP(resp, req)

				So(resp.Code, ShouldEqual, updateErr.Status())
				body, isJSON := getJSON(resp.HeaderMap, resp.Body)
				So(isJSON, ShouldBeTrue)
				So(body.String(), ShouldEqual, `"Could not update channel version"`)
			})
		})

		Convey("Should always route updates if `AlwaysRoute` is enabled", func() {
			eh := NewEndpointHandler()
			eh.setApp(app)
			eh.alwaysRoute = true
			app.SetEndpointHandler(eh)

			uaid := "6952a68ee0e7444ebc54f935c4444b13"
			app.AddWorker(uaid, mckWorker)

			chid := "b7ede546585f4cc9b95e9340e3406951"
			version := int64(1)
			data := "Happy, happy, joy, joy!"

			Convey("And router delivery fails, local succeeds", func() {
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, version,
						gomock.Any(), "", data).Return(false, nil),
					mckStat.EXPECT().Increment("router.broadcast.miss"),
					mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
					mckWorker.EXPECT().Send(chid, version, data).Return(nil),
					mckStat.EXPECT().Increment("updates.appserver.received"),
				)

				ok := eh.deliver(nil, uaid, chid, version, "", data)
				So(ok, ShouldBeTrue)
			})

			Convey("And router delivery succeeds, local succeeds", func() {
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, version,
						gomock.Any(), "", data).Return(true, nil),
					mckStat.EXPECT().Increment("router.broadcast.hit"),
					mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
					mckWorker.EXPECT().Send(chid, version, data).Return(nil),
					mckStat.EXPECT().Increment("updates.appserver.received"),
				)

				ok := eh.deliver(nil, uaid, chid, version, "", data)
				So(ok, ShouldBeTrue)
			})

			Convey("And router delivery succeeds, local fails", func() {
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, version,
						gomock.Any(), "", data).Return(true, nil),
					mckStat.EXPECT().Increment("router.broadcast.hit"),
					mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
					mckWorker.EXPECT().Send(chid, version, data).Return(
						errors.New("client gone")),
					mckStat.EXPECT().Increment("updates.appserver.received"),
				)

				ok := eh.deliver(nil, uaid, chid, version, "", data)
				So(ok, ShouldBeTrue)
			})

			Convey("And router/local delivery fails", func() {
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, version,
						gomock.Any(), "", data).Return(false, nil),
					mckStat.EXPECT().Increment("router.broadcast.miss"),
					mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
					mckWorker.EXPECT().Send(chid, version, data).Return(
						errors.New("client gone")),
					mckStat.EXPECT().Increment("updates.appserver.rejected"),
				)

				ok := eh.deliver(nil, uaid, chid, version, "", data)
				So(ok, ShouldBeFalse)
			})

		})
	})
}
Esempio n. 21
0
func HttpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)

	json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`

	c.Specify("A HttpInput", func() {

		httpInput := HttpInput{}
		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
		ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

		runOutputChan := make(chan error, 1)
		startInput := func() {
			go func() {
				runOutputChan <- httpInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
		}

		ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

		// These assume that every sub-spec starts the input.
		config := httpInput.ConfigStruct().(*HttpInputConfig)
		tickChan := make(chan time.Time)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
		ith.MockHelper.EXPECT().Hostname().Return("hekatests.example.com")

		// These assume that every sub-spec makes exactly one HTTP request.
		ith.MockInputRunner.EXPECT().NewSplitterRunner("0").Return(ith.MockSplitterRunner)
		getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
		getRecCall.Return(len(json_post), []byte(json_post), io.EOF)
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		decChan := make(chan func(*PipelinePack), 1)
		packDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
		packDecCall.Do(func(dec func(*PipelinePack)) {
			decChan <- dec
		})
		ith.MockSplitterRunner.EXPECT().DeliverRecord([]byte(json_post), nil)
		ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()
		splitter := &TokenSplitter{} // not actually used
		ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)

		c.Specify("honors time ticker to flush", func() {
			// Spin up a http server.
			server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876)
			c.Expect(err, gs.IsNil)
			go server.Start("/")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9876/"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			// Getting the decorator means we've made our HTTP request.
			<-decChan
		})

		c.Specify("supports configuring HTTP Basic Authentication", func() {
			// Spin up a http server which expects username "user" and password "password"
			server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875)
			c.Expect(err, gs.IsNil)
			go server.Start("/BasicAuthTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9875/BasicAuthTest"
			config.User = "******"
			config.Password = "******"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring a different HTTP method", func() {
			// Spin up a http server which expects requests with method "POST"
			server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874)
			c.Expect(err, gs.IsNil)
			go server.Start("/PostTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9874/PostTest"
			config.Method = "POST"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring HTTP headers", func() {
			// Spin up a http server which expects requests with method "POST"
			server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873)
			c.Expect(err, gs.IsNil)
			go server.Start("/HeadersTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9873/HeadersTest"
			config.Headers = map[string]string{"Accept": "text/plain"}

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring a request body", func() {
			// Spin up a http server that echoes back the request body
			server, err := plugins_ts.NewHttpBodyServer("localhost", 9872)
			c.Expect(err, gs.IsNil)
			go server.Start("/BodyTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9872/BodyTest"
			config.Method = "POST"
			config.Body = json_post

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			respBodyChan := make(chan []byte, 1)
			getRecCall.Do(func(r io.Reader) {
				respBody := make([]byte, len(config.Body))
				n, err := r.Read(respBody)
				c.Expect(n, gs.Equals, len(config.Body))
				c.Expect(err, gs.Equals, io.EOF)
				respBodyChan <- respBody
			})

			startInput()
			tickChan <- time.Now()

			respBody := <-respBodyChan
			c.Expect(string(respBody), gs.Equals, json_post)
		})

		httpInput.Stop()
		runOutput := <-runOutputChan
		c.Expect(runOutput, gs.IsNil)
	})
}
Esempio n. 22
0
func InputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	var wg sync.WaitGroup
	errChan := make(chan error, 1)
	tickChan := make(chan time.Time)
	defer func() {
		close(tickChan)
		close(errChan)
		ctrl.Finish()
	}()

	pConfig := NewPipelineConfig(nil)
	c.Specify("A SandboxInput", func() {
		input := new(SandboxInput)
		input.SetPipelineConfig(pConfig)

		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
		ith.PackSupply = make(chan *PipelinePack, 1)
		ith.Pack = NewPipelinePack(ith.PackSupply)
		ith.PackSupply <- ith.Pack

		startInput := func() {
			wg.Add(1)
			go func() {
				errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
				wg.Done()
			}()
		}

		c.Specify("test a polling input", func() {
			tickChan := time.Tick(10 * time.Millisecond)
			ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(2)
			ith.MockInputRunner.EXPECT().LogError(fmt.Errorf("failure message"))

			var cnt int
			ith.MockInputRunner.EXPECT().Inject(gomock.Any()).Do(
				func(pack *PipelinePack) {
					switch cnt {
					case 0:
						c.Expect(pack.Message.GetPayload(), gs.Equals, "line 1")
					case 1:
						c.Expect(pack.Message.GetPayload(), gs.Equals, "line 3")
						input.Stop()
					}
					cnt++
					ith.PackSupply <- pack
				}).Times(2)

			config := input.ConfigStruct().(*sandbox.SandboxConfig)
			config.ScriptFilename = "../lua/testsupport/input.lua"

			err := input.Init(config)
			c.Assume(err, gs.IsNil)

			startInput()

			wg.Wait()
			c.Expect(<-errChan, gs.IsNil)
			c.Expect(input.processMessageCount, gs.Equals, int64(2))
			c.Expect(input.processMessageFailures, gs.Equals, int64(1))
			c.Expect(input.processMessageBytes, gs.Equals, int64(72))
		})

		c.Specify("run once input", func() {
			var tickChan <-chan time.Time
			ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(1)
			ith.MockInputRunner.EXPECT().LogMessage("single run completed")
			ith.MockInputRunner.EXPECT().Inject(gomock.Any()).Do(
				func(pack *PipelinePack) {
					c.Expect(pack.Message.GetPayload(), gs.Equals, "line 1")
					ith.PackSupply <- pack
				}).Times(1)

			config := input.ConfigStruct().(*sandbox.SandboxConfig)
			config.ScriptFilename = "../lua/testsupport/input.lua"

			err := input.Init(config)
			c.Assume(err, gs.IsNil)

			startInput()

			wg.Wait()
			c.Expect(<-errChan, gs.IsNil)
			c.Expect(input.processMessageCount, gs.Equals, int64(1))
			c.Expect(input.processMessageBytes, gs.Equals, int64(36))
		})

		c.Specify("exit with error", func() {
			tickChan := make(chan time.Time)
			defer close(tickChan)
			ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
			ith.MockInputRunner.EXPECT().LogError(fmt.Errorf("process_message() ../lua/testsupport/input_error.lua:2: boom"))

			config := input.ConfigStruct().(*sandbox.SandboxConfig)
			config.ScriptFilename = "../lua/testsupport/input_error.lua"

			err := input.Init(config)
			c.Assume(err, gs.IsNil)

			startInput()

			wg.Wait()
			c.Expect(<-errChan, gs.IsNil)
			c.Expect(input.processMessageCount, gs.Equals, int64(0))
			c.Expect(input.processMessageBytes, gs.Equals, int64(0))
		})
	})
}
Esempio n. 23
0
func TestEndpointPinger(t *testing.T) {
	useMockFuncs()
	defer useStdFuncs()

	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()

	mckStat := NewMockStatistician(mockCtrl)
	mckPinger := NewMockPropPinger(mockCtrl)
	mckStore := NewMockStore(mockCtrl)
	mckWorker := NewMockWorker(mockCtrl)

	Convey("Proprietary pings", t, func() {
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)
		app.SetPropPinger(mckPinger)
		app.SetStore(mckStore)

		eh := NewEndpointHandler()
		eh.setApp(app)
		eh.setMaxDataLen(4096)
		app.SetEndpointHandler(eh)

		Convey("Should return early if the pinger can bypass the WebSocket", func() {
			uaid := "91357e1a34714cadacb3f13cf47a2736"
			app.AddWorker(uaid, mckWorker)

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   nil,
			}
			gomock.InOrder(
				mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
				mckStat.EXPECT().Increment("updates.appserver.incoming"),
				mckPinger.EXPECT().Send(uaid, int64(1257894000), "").Return(true, nil),
				mckPinger.EXPECT().CanBypassWebsocket().Return(true),
				mckStat.EXPECT().Increment("updates.appserver.received"),
				mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
			)
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 200)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, "{}")
		})

		Convey("Should continue if the pinger cannot bypass the WebSocket", func() {
			uaid := "e3fc2cf1dc44424685010148b076d08b"
			app.AddWorker(uaid, mckWorker)

			data := randomText(eh.maxDataLen)
			vals := make(url.Values)
			vals.Set("data", data)

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}
			gomock.InOrder(
				mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
				mckStat.EXPECT().Increment("updates.appserver.incoming"),
				mckPinger.EXPECT().Send(uaid, int64(1257894000), data).Return(true, nil),
				mckPinger.EXPECT().CanBypassWebsocket().Return(false),
				mckStore.EXPECT().Update(uaid, "456", int64(1257894000)),
				mckWorker.EXPECT().Send("456", int64(1257894000), data),
				mckStat.EXPECT().Increment("updates.appserver.received"),
				mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
			)
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 200)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, "{}")
		})

		Convey("Should continue if the pinger fails", func() {
			uaid := "8f412f5cb2384183bf60f7da26737271"
			app.AddWorker(uaid, mckWorker)

			vals := make(url.Values)
			vals.Set("version", "7")
			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}
			gomock.InOrder(
				mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
				mckStat.EXPECT().Increment("updates.appserver.incoming"),
				mckPinger.EXPECT().Send(uaid, int64(7), "").Return(
					true, errors.New("oops")),
				mckStore.EXPECT().Update(uaid, "456", int64(7)),
				mckWorker.EXPECT().Send("456", int64(7), ""),
				mckStat.EXPECT().Increment("updates.appserver.received"),
				mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
			)
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 200)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, "{}")
		})
	})
}
Esempio n. 24
0
func TestSocketListenerConfig(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()
	mckStat := NewMockStatistician(mockCtrl)

	tests := []struct {
		name     string
		hostname string
		conf     listenerConfig
		url      string
		ok       bool
	}{
		{
			name:     "Invalid address, TLS, default hostname",
			hostname: "example.com",
			conf: listenerConfig{
				listener: newMockListener(netAddr{"test", "!@#$"}),
				useTLS:   true,
				maxConns: 5,
			},
			url: "wss://example.com",
			ok:  true,
		},
		{
			name:     "IPv6 address, default hostname",
			hostname: "localhost",
			conf: listenerConfig{
				listener: newMockListener(netAddr{"test", "[::1]:8080"}),
				useTLS:   false,
				maxConns: 1,
			},
			url: "ws://localhost:8080",
			ok:  true,
		},
		{
			name: "IPv4 address, TLS, no default hostname",
			conf: listenerConfig{
				listener: newMockListener(netAddr{"test", "127.0.0.1:8090"}),
				useTLS:   true,
				maxConns: 1,
			},
			url: "wss://127.0.0.1:8090",
			ok:  true,
		},
	}
	for _, test := range tests {
		func() {
			app := NewApplication()
			if len(test.hostname) > 0 {
				app.hostname = test.hostname
			}
			app.SetLogger(mckLogger)
			app.SetMetrics(mckStat)
			sh := NewSocketHandler()
			defer sh.Close()
			sh.setApp(app)
			err := sh.listenWithConfig(test.conf)
			if err != nil {
				if test.ok {
					t.Errorf("On test %s, got listener error: %s", test.name, err)
				}
				return
			}
			if !test.ok {
				listener := sh.Listener()
				t.Errorf("On test %s, got %#v; want listener error", test.name, listener)
				listener.Close()
				return
			}
			if actual := sh.URL(); actual != test.url {
				t.Errorf("Mismatched handler URL: got %q; want %q",
					actual, test.url)
			}
			if actual := sh.MaxConns(); actual != test.conf.maxConns {
				t.Errorf("Mismatched maximum connection count: got %d; want %d",
					actual, test.conf.maxConns)
			}
		}()
	}
}
Esempio n. 25
0
func TestEndpointResolveKey(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()

	mckStat := NewMockStatistician(mockCtrl)
	mckStore := NewMockStore(mockCtrl)

	Convey("Endpoint tokens", t, func() {
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)
		app.SetStore(mckStore)

		Convey("Should return a 404 for invalid tokens", func() {
			app.SetTokenKey("c3v0AlmmxXu_LSfdZY3l3eayLsIwkX48")
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL: &url.URL{
					Path: "/update/j1bqzFq9WiwFZbqay-y7xVlfSvtO1eY="}, // "123.456"
			}
			gomock.InOrder(
				mckStore.EXPECT().KeyToIDs("123.456").Return("", "", ErrInvalidKey),
				mckStat.EXPECT().Increment("updates.appserver.invalid"),
			)
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 404)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Invalid Token"`)
		})

		Convey("Should not decode plaintext tokens without a key", func() {
			var err error

			app.SetTokenKey("")
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			_, err = eh.decodePK("")
			So(err, ShouldNotBeNil)

			pk, err := eh.decodePK("123.456")
			So(pk, ShouldEqual, "123.456")
		})

		Convey("Should normalize decoded tokens", func() {
			app.SetTokenKey("LM1xDImCx0rB46LCnx-3v4-Iyfk1LeKJbx9wuvx_z3U=")
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			// Hyphenated IDs should be normalized.
			uaid := "dbda2ba2-004c-491f-9e3d-c5950aee93de"
			chid := "848cd568-3f2a-4108-9ce4-bd0d928ecad4"
			// " \t%s.%s\r\n" % (uaid, chid)
			encodedKey := "qfGSdZzwf20GXiYubmZfIXj11Rx4RGJujFsjSQGdF4LRBhHbB_vt3hdW7cRvL9Fq_t_guMBGkDgebOoa5gRd1GGLN-Cv6h5hkpRTbdju8Tk-hMyC91BP4CEres_8"

			// decodePK should trim whitespace from encoded keys.
			mckStore.EXPECT().KeyToIDs(
				fmt.Sprintf("%s.%s", uaid, chid)).Return(uaid, chid, nil)
			actualUAID, actualCHID, err := eh.resolvePK(encodedKey)
			So(err, ShouldBeNil)
			So(actualUAID, ShouldEqual, uaid)
			So(actualCHID, ShouldEqual, chid)
		})

		Convey("Should reject invalid tokens", func() {
			var err error

			app.SetTokenKey("IhnNwMNbsFWiafTXSgF4Ag==")
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			invalidKey := "b54QOw2omSWBiEq0IuyfBGxHBIR7AI9YhCMA0lP9" // "_=!@#$%^&*()[]"
			uaid := "82398a648c834f8b838cb3945eceaf29"
			chid := "af445ad07e5f46b7a6c858150fc5aa92"
			validKey := fmt.Sprintf("%s.%s", uaid, chid)
			encodedKey := "swKSH8P2qprRt5y0J4Wi7ybl-qzFv1j09WPOfuabpEJmVUqwUpxjprXc2R3Yw0ITbqc_Swntw9_EpCgo_XuRTn7Q7opQYoQUgMPhCgT0EGbK"

			_, _, err = eh.resolvePK(invalidKey[:8])
			So(err, ShouldNotBeNil)

			_, _, err = eh.resolvePK(invalidKey)
			So(err, ShouldNotBeNil)

			// Reject plaintext tokens if a key is specified.
			_, _, err = eh.resolvePK(validKey)
			So(err, ShouldNotBeNil)

			mckStore.EXPECT().KeyToIDs(validKey).Return("", "", ErrInvalidKey)
			_, _, err = eh.resolvePK(encodedKey)
			So(err, ShouldNotBeNil)

			mckStore.EXPECT().KeyToIDs(validKey).Return(uaid, chid, nil)
			actualUAID, actualCHID, err := eh.resolvePK(encodedKey)
			So(err, ShouldBeNil)
			So(actualUAID, ShouldEqual, uaid)
			So(actualCHID, ShouldEqual, chid)
		})
	})
}
Esempio n. 26
0
func MultiDecoderSpec(c gospec.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil) // initializes Globals()

	c.Specify("A MultiDecoder", func() {
		subsTOML := `[StartsWithM]
		type = "PayloadRegexDecoder"
		match_regex = '^(?P<TheData>m.*)'
		log_errors = true
		[StartsWithM.message_fields]
		StartsWithM = "%TheData%"
	
		[StartsWithS]
		type = "PayloadRegexDecoder"
		match_regex = '^(?P<TheData>s.*)'
		log_errors = true
		[StartsWithS.message_fields]
		StartsWithS = "%TheData%"
	
		[StartsWithM2]
		type = "PayloadRegexDecoder"
		match_regex = '^(?P<TheData>m.*)'
		log_errors = true
		[StartsWithM2.message_fields]
		StartsWithM2 = "%TheData%"
		`

		RegisterPlugin("PayloadRegexDecoder", func() interface{} {
			return &PayloadRegexDecoder{}
		})
		defer delete(AvailablePlugins, "PayloadRegexDecoder")

		var configFile ConfigFile
		_, err := toml.Decode(subsTOML, &configFile)
		c.Assume(err, gs.IsNil)

		decoder := new(MultiDecoder)
		decoder.SetName("MyMultiDecoder")
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*MultiDecoderConfig)

		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		mSection, ok := configFile["StartsWithM"]
		c.Assume(ok, gs.IsTrue)
		mMaker, err := NewPluginMaker("StartsWithM", pConfig, mSection)
		c.Assume(err, gs.IsNil)
		pConfig.DecoderMakers["StartsWithM"] = mMaker

		conf.Subs = []string{"StartsWithM"}
		errMsg := "All subdecoders failed."

		dRunner := pipelinemock.NewMockDecoderRunner(ctrl)
		// An error will be spit out b/c there's no real *dRunner in there;
		// doesn't impact the tests.
		dRunner.EXPECT().LogError(gomock.Any())

		c.Specify("decodes simple messages", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "matching text"
			pack.Message.SetPayload(regex_data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			value, ok := pack.Message.GetFieldValue("StartsWithM")
			c.Assume(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, regex_data)
		})

		c.Specify("returns an error if all decoders fail", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "non-matching text"
			pack.Message.SetPayload(regex_data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, errMsg)
		})

		c.Specify("logs subdecoder failures when configured to do so", func() {
			conf.LogSubErrors = true
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "non-matching text"
			pack.Message.SetPayload(regex_data)

			// Expect that we log an error for undecoded message.
			dRunner.EXPECT().LogError(fmt.Errorf(
				"Subdecoder 'StartsWithM' decode error: No match: %s", regex_data)).AnyTimes()

			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, errMsg)
		})

		c.Specify("sets subdecoder runner correctly", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			// Call LogError to appease the angry gomock gods.
			dRunner.LogError(errors.New("foo"))

			// Now create a real *dRunner, pass it in, make sure a wrapper
			// gets handed to the subdecoder.
			dr := NewDecoderRunner(decoder.Name, decoder, 10)
			decoder.SetDecoderRunner(dr)
			sub := decoder.Decoders[0]
			subRunner := sub.(*PayloadRegexDecoder).dRunner
			c.Expect(subRunner.Name(), gs.Equals,
				fmt.Sprintf("%s-StartsWithM", decoder.Name))
			c.Expect(subRunner.Decoder(), gs.Equals, sub)
		})

		c.Specify("with multiple registered decoders", func() {
			sSection, ok := configFile["StartsWithS"]
			c.Assume(ok, gs.IsTrue)
			sMaker, err := NewPluginMaker("StartsWithS", pConfig, sSection)
			c.Assume(err, gs.IsNil)
			pConfig.DecoderMakers["StartsWithS"] = sMaker

			m2Section, ok := configFile["StartsWithM2"]
			c.Assume(ok, gs.IsTrue)
			m2Maker, err := NewPluginMaker("StartsWithM2", pConfig, m2Section)
			c.Assume(err, gs.IsNil)
			pConfig.DecoderMakers["StartsWithM2"] = m2Maker

			conf.Subs = append(conf.Subs, "StartsWithS", "StartsWithM2")

			// Two more subdecoders means two more LogError calls.
			dRunner.EXPECT().LogError(gomock.Any()).Times(2)

			c.Specify("defaults to `first-wins` cascading", func() {
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)

				c.Specify("on a first match condition", func() {
					pack.Message.SetPayload("match first")
					_, err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("and a second match condition", func() {
					pack.Message.SetPayload("second match")
					_, err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("returning an error if they all fail", func() {
					pack.Message.SetPayload("won't match")
					packs, err := decoder.Decode(pack)
					c.Expect(len(packs), gs.Equals, 0)
					c.Expect(err.Error(), gs.Equals, errMsg)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})
			})

			c.Specify("and using `all` cascading", func() {
				conf.CascadeStrategy = "all"
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)

				c.Specify("matches multiples when appropriate", func() {
					pack.Message.SetPayload("matches twice")
					_, err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsTrue)
				})

				c.Specify("matches singles when appropriate", func() {
					pack.Message.SetPayload("second match")
					_, err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("returns an error if they all fail", func() {
					pack.Message.SetPayload("won't match")
					packs, err := decoder.Decode(pack)
					c.Expect(len(packs), gs.Equals, 0)
					c.Expect(err.Error(), gs.Equals, errMsg)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})
			})
		})
	})

	c.Specify("A MultiDecoder w/ MultiOutput", func() {
		subsTOML := `[sub0]
		type = "MultiOutputDecoder"

		[sub1]
		type = "MultiOutputDecoder"

		[sub2]
		type = "MultiOutputDecoder"
		`

		RegisterPlugin("MultiOutputDecoder", func() interface{} {
			return &MultiOutputDecoder{}
		})
		defer delete(AvailablePlugins, "MultiOutputDecoder")

		var configFile ConfigFile
		_, err := toml.Decode(subsTOML, &configFile)

		decoder := new(MultiDecoder)
		decoder.SetName("MyMultiDecoder")
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*MultiDecoderConfig)
		conf.CascadeStrategy = "all"

		supply := make(chan *PipelinePack, 10)
		pack := NewPipelinePack(supply)

		sub0Section, ok := configFile["sub0"]
		c.Assume(ok, gs.IsTrue)
		sub0Maker, err := NewPluginMaker("sub0", pConfig, sub0Section)
		c.Assume(err, gs.IsNil)
		pConfig.DecoderMakers["sub0"] = sub0Maker

		sub1Section, ok := configFile["sub1"]
		c.Assume(ok, gs.IsTrue)
		sub1Maker, err := NewPluginMaker("sub1", pConfig, sub1Section)
		c.Assume(err, gs.IsNil)
		pConfig.DecoderMakers["sub1"] = sub1Maker

		sub2Section, ok := configFile["sub2"]
		c.Assume(ok, gs.IsTrue)
		sub2Maker, err := NewPluginMaker("sub2", pConfig, sub2Section)
		c.Assume(err, gs.IsNil)
		pConfig.DecoderMakers["sub2"] = sub2Maker

		conf.Subs = []string{"sub0", "sub1", "sub2"}
		dRunner := pipelinemock.NewMockDecoderRunner(ctrl)

		c.Specify("always tries to decode all packs", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			decoder.SetDecoderRunner(dRunner)

			packs, err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(len(packs), gs.Equals, 15)
		})
	})
}
Esempio n. 27
0
func TestLocatorReadyNotify(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	uaid := "fce61180716a40ed8e79bf5ff0ba34bc"

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(),
		gomock.Any()).AnyTimes()

	var (
		// routerPipes maps fake peer addresses to their respective pipes. Used
		// by dialRouter to connect to peers.
		routerPipes = make(map[netAddr]*pipeListener)

		// contacts is a list of peer URLs for the locator.
		contacts []string
	)

	// Fake listener for the sender's router, used to test self-routing.
	sndRouterAddr := netAddr{"tcp", "snd-router.example.com:3000"}
	sndRouterPipe := newPipeListener()
	defer sndRouterPipe.Close()
	routerPipes[sndRouterAddr] = sndRouterPipe
	contacts = append(contacts, "http://snd-router.example.com:3000")

	// Fake listener for the receiver's router, used to test routing updates
	// to different hosts.
	recvRouterAddr := netAddr{"tcp", "recv-router.example.com:3000"}
	recvRouterPipe := newPipeListener()
	defer recvRouterPipe.Close()
	routerPipes[recvRouterAddr] = recvRouterPipe
	contacts = append(contacts, "http://recv-router.example.com:3000")

	// Fake listener for the receiver's WebSocket handler, used to accept a
	// WebSocket client connection.
	socketHandlerPipe := newPipeListener()
	defer socketHandlerPipe.Close()

	// Fake locator.
	mckLocator := NewMockLocator(mockCtrl)
	mckLocator.EXPECT().Contacts(uaid).Return(contacts, nil).Times(2)

	// Fake dialer to connect to each peer's routing listener.
	dialRouter := func(network, address string) (net.Conn, error) {
		if pipe, ok := routerPipes[netAddr{network, address}]; ok {
			return pipe.Dial(network, address)
		}
		return nil, &netErr{temporary: false, timeout: false}
	}
	// Configures a fake router for the app.
	setRouter := func(app *Application, listener net.Listener) {
		r := NewBroadcastRouter()
		r.setApp(app)
		r.setClientOptions(10, 3*time.Second, 3*time.Second) // Defaults.
		r.setClientTransport(&http.Transport{Dial: dialRouter})
		r.listenWithConfig(listenerConfig{listener: listener})
		r.maxDataLen = 4096
		r.server = newServeWaiter(&http.Server{Handler: r.ServeMux()})
		app.SetRouter(r)
	}

	// sndApp is the server broadcasting the update. The locator returns the
	// addresses of the sender and receiver to test self-routing.
	sndApp := NewApplication()
	sndApp.SetLogger(mckLogger)
	sndStat := NewMockStatistician(mockCtrl)
	sndApp.SetMetrics(sndStat)
	sndStore := NewMockStore(mockCtrl)
	sndApp.SetStore(sndStore)
	sndApp.SetLocator(mckLocator)
	// Set up a fake router for the sender.
	setRouter(sndApp, sndRouterPipe)

	// recvApp is the server receiving the update.
	recvApp := NewApplication()
	recvApp.SetLogger(mckLogger)
	recvStat := NewMockStatistician(mockCtrl)
	recvApp.SetMetrics(recvStat)
	recvStore := NewMockStore(mockCtrl)
	recvApp.SetStore(recvStore)
	// Wrap the fake locator in a type that implements ReadyNotifier.
	recvLocator := newMockReadyNotifier(mckLocator)
	recvApp.SetLocator(recvLocator)
	// Set up a fake WebSocket handler for the receiver.
	recvSocketHandler := NewSocketHandler()
	recvSocketHandler.setApp(recvApp)
	recvSocketHandler.listenWithConfig(listenerConfig{
		listener: socketHandlerPipe})
	recvSocketHandler.server = newServeWaiter(&http.Server{Handler: recvSocketHandler.ServeMux()})
	recvApp.SetSocketHandler(recvSocketHandler)
	// Set up a fake router for the receiver.
	setRouter(recvApp, recvRouterPipe)

	chid := "2b7c5c27d6224bfeaf1c158c3c57fca3"
	version := int64(2)
	data := "I'm a little teapot, short and stout."

	var wg sync.WaitGroup // Waits for the client to close.
	wg.Add(1)
	dialChan := make(chan bool) // Signals when the client connects.
	timeout := closeAfter(2 * time.Second)

	go func() {
		defer wg.Done()
		origin := &url.URL{Scheme: "ws", Host: "recv-conn.example.com"}
		ws, err := dialSocketListener(socketHandlerPipe, &websocket.Config{
			Location: origin,
			Origin:   origin,
			Version:  websocket.ProtocolVersionHybi13,
		})
		if err != nil {
			t.Errorf("Error dialing host: %s", err)
			return
		}
		defer ws.Close()
		err = websocket.JSON.Send(ws, struct {
			Type       string   `json:"messageType"`
			DeviceID   string   `json:"uaid"`
			ChannelIDs []string `json:"channelIDs"`
		}{"hello", uaid, []string{}})
		if err != nil {
			t.Errorf("Error writing handshake request: %s", err)
			return
		}
		helloReply := new(HelloReply)
		if err = websocket.JSON.Receive(ws, helloReply); err != nil {
			t.Errorf("Error reading handshake reply: %s", err)
			return
		}
		select {
		case dialChan <- true:
		case <-timeout:
			t.Errorf("Timed out waiting for router")
			return
		}
		flushReply := new(FlushReply)
		if err = websocket.JSON.Receive(ws, flushReply); err != nil {
			t.Errorf("Error reading routed update: %s", err)
			return
		}
		ok := false
		expected := Update{chid, uint64(version), data}
		for _, update := range flushReply.Updates {
			if ok = update == expected; ok {
				break
			}
		}
		if !ok {
			t.Errorf("Missing update %#v in %#v", expected, flushReply.Updates)
			return
		}
	}()

	// Start the handlers.
	errChan := make(chan error, 3)
	go sndApp.Router().Start(errChan)
	go recvApp.SocketHandler().Start(errChan)
	go recvApp.Router().Start(errChan)

	// First and second routing attempts to self.
	sndStat.EXPECT().Increment("updates.routed.unknown").Times(2)

	// Initial routing attempt to peer.
	recvStat.EXPECT().Increment("updates.routed.unknown")
	// Client connects to peer.
	recvStat.EXPECT().Increment("client.socket.connect")
	recvStore.EXPECT().CanStore(0).Return(true)
	recvStat.EXPECT().Increment("updates.client.hello")
	recvStore.EXPECT().FetchAll(uaid, gomock.Any()).Return(nil, nil, nil)
	recvStat.EXPECT().Timer("client.flush", gomock.Any())
	// Second routing attempt to peer.
	recvStat.EXPECT().Increment("updates.routed.incoming")
	recvStat.EXPECT().Increment("updates.sent")
	recvStat.EXPECT().Timer("client.flush", gomock.Any())
	recvStat.EXPECT().Increment("updates.routed.received")
	recvStat.EXPECT().Timer("client.socket.lifespan", gomock.Any())
	recvStat.EXPECT().Increment("client.socket.disconnect")

	// Initial routing attempt should fail; the WebSocket listener shouldn't
	// accept client connections before the locator is ready.
	delivered, err := sndApp.Router().Route(nil, uaid, chid, version, timeNow(),
		"disconnected", data)
	if err != nil {
		t.Errorf("Error routing to disconnected client: %s", err)
	} else if delivered {
		t.Error("Should not route to disconnected client")
	}
	// Signal the locator is ready, then wait for the client to connect.
	recvLocator.SignalReady()
	select {
	case <-dialChan:
	case <-time.After(5 * time.Second):
		t.Fatalf("Timed out waiting for the client to connect")
	}
	// Routing should succeed once the client is connected.
	delivered, err = sndApp.Router().Route(nil, uaid, chid, version, timeNow(),
		"connected", data)
	if err != nil {
		t.Errorf("Error routing to connected client: %s", err)
	} else if !delivered {
		t.Error("Should route to connected client")
	}

	gomock.InOrder(
		mckLocator.EXPECT().Close(),
		recvStore.EXPECT().Close(),
	)
	if err := recvApp.Close(); err != nil {
		t.Errorf("Error closing peer: %s", err)
	}
	wg.Wait()
	gomock.InOrder(
		mckLocator.EXPECT().Close(),
		sndStore.EXPECT().Close(),
	)
	if err := sndApp.Close(); err != nil {
		t.Errorf("Error closing self: %s", err)
	}
	// Wait for the handlers to stop.
	for i := 0; i < 3; i++ {
		<-errChan
	}
}
Esempio n. 28
0
func HttpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	var (
		reqMethod string
		reqBody   string
		reqHeader http.Header
		handleWg  sync.WaitGroup
		runWg     sync.WaitGroup
		delay     bool
	)

	handler := new(_test_handler)
	handler.serveHttp = func(rw http.ResponseWriter, req *http.Request) {
		defer handleWg.Done()
		if delay {
			time.Sleep(time.Duration(2) * time.Millisecond)
		}
		// Capture request body for test assertions.
		p := make([]byte, req.ContentLength)
		_, err := req.Body.Read(p)
		c.Expect(err.Error(), gs.Equals, "EOF")
		reqBody = string(p)
		// Capture request method and headers for test assertions.
		reqMethod = req.Method
		reqHeader = req.Header
		// Respond with either a response body or a response code.
		if len(handler.respBody) > 0 {
			rw.Write([]byte(handler.respBody))
		} else {
			rw.WriteHeader(handler.respCode)
		}
		flusher := rw.(http.Flusher)
		flusher.Flush()
	}

	oth := ts.NewOutputTestHelper(ctrl)
	encoder := new(plugins.PayloadEncoder)
	encConfig := new(plugins.PayloadEncoderConfig)
	err := encoder.Init(encConfig)
	c.Expect(err, gs.IsNil)
	inChan := make(chan *pipeline.PipelinePack, 1)
	recycleChan := make(chan *pipeline.PipelinePack, 1)
	pack := pipeline.NewPipelinePack(recycleChan)
	pack.Message = pipeline_ts.GetTestMessage()

	c.Specify("An HttpOutput", func() {
		httpOutput := new(HttpOutput)
		config := httpOutput.ConfigStruct().(*HttpOutputConfig)

		c.Specify("barfs on bogus URLs", func() {
			config.Address = "one-two-three-four"
			err := httpOutput.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("that is started", func() {
			server := httptest.NewServer(handler)
			defer server.Close()

			runOutput := func() {
				httpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				runWg.Done()
			}

			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
			payload := "this is the payload"
			pack.Message.SetPayload(payload)
			oth.MockOutputRunner.EXPECT().Encode(gomock.Any()).Return(
				[]byte(payload), nil)
			config.Address = server.URL
			handler.respBody = "Response Body"

			c.Specify("makes http POST requests by default", func() {
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(reqMethod, gs.Equals, "POST")
			})

			c.Specify("makes http PUT requests", func() {
				config.Method = "put"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(reqMethod, gs.Equals, "PUT")
			})

			c.Specify("makes http GET requests", func() {
				config.Method = "get"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, "")
				c.Expect(reqMethod, gs.Equals, "GET")
			})

			c.Specify("correctly passes headers along", func() {
				config.Headers = http.Header{
					"One":  []string{"two", "three"},
					"Four": []string{"five", "six", "seven"},
				}
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(reqBody, gs.Equals, payload)
				c.Expect(len(reqHeader["One"]), gs.Equals, len(config.Headers["One"]))
				c.Expect(len(reqHeader["Four"]), gs.Equals, len(config.Headers["Four"]))
			})

			c.Specify("uses http auth when specified", func() {
				config.Username = "******"
				config.Password = "******"
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				auth := reqHeader.Get("Authorization")
				c.Expect(strings.HasPrefix(auth, "Basic "), gs.IsTrue)
				decodedAuth, err := base64.StdEncoding.DecodeString(auth[6:])
				c.Expect(err, gs.IsNil)
				c.Expect(string(decodedAuth), gs.Equals, "user:pass")
			})

			c.Specify("logs error responses", func() {
				handler.respBody = ""
				handler.respCode = 500
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)

				var errMsg string
				oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
					func(err error) {
						errMsg = err.Error()
					})
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(strings.HasPrefix(errMsg,
					"HTTP Error code returned: 500"), gs.IsTrue)
			})

			c.Specify("honors http timeout interval", func() {
				config.HttpTimeout = 1 // 1 millisecond
				err := httpOutput.Init(config)
				c.Expect(err, gs.IsNil)

				var errMsg string
				oth.MockOutputRunner.EXPECT().LogError(gomock.Any()).Do(
					func(err error) {
						errMsg = err.Error()
					})
				delay = true
				runWg.Add(1)
				go runOutput()
				handleWg.Add(1)
				inChan <- pack
				close(inChan)
				handleWg.Wait()
				runWg.Wait()
				c.Expect(strings.Contains(errMsg, "use of closed network connection"),
					gs.IsTrue)
			})
		})
	})
}
Esempio n. 29
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	config := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: config}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}
		lsiConfig.Decoder = "decoder-name"

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)
			mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
				// Expect calls to get decoder and decode each message. Since the
				// decoding is a no-op, the message payload will be the log file
				// line, unchanged.
				pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
					"-"+lsiConfig.Decoder)
				pbcall.Return(mockDecoderRunner, true)

				decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
				decodeCall.Return(ith.DecodeChan)

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				d, _ := time.ParseDuration("5s")
				timeout := time.After(d)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case <-ith.DecodeChan:
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Esempio n. 30
0
func TcpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A TcpInput", func() {
		tcpInput := &TcpInput{}
		config := &TcpInputConfig{
			Net:     "tcp",
			Address: ith.AddrStr,
		}

		bytesChan := make(chan []byte, 1)
		errChan := make(chan error, 1)
		var srDoneWG sync.WaitGroup

		startServer := func() {
			srDoneWG.Add(1)
			ith.MockInputRunner.EXPECT().Name().Return("mock_name")
			ith.MockInputRunner.EXPECT().NewDeliverer(gomock.Any()).Return(ith.MockDeliverer)
			ith.MockDeliverer.EXPECT().Done()
			ith.MockInputRunner.EXPECT().NewSplitterRunner(gomock.Any()).Return(
				ith.MockSplitterRunner)
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			ith.MockSplitterRunner.EXPECT().Done().Do(func() {
				srDoneWG.Done()
			})

			// splitCall gets called twice. The first time it returns nil, the
			// second time it returns io.EOF.
			splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
				ith.MockDeliverer).AnyTimes()
			splitCall.Do(func(conn net.Conn, del Deliverer) {
				recd, _ := ioutil.ReadAll(conn)
				bytesChan <- recd
				splitCall.Return(io.EOF)
			})

			err := tcpInput.Run(ith.MockInputRunner, ith.MockHelper)
			errChan <- err
		}

		c.Specify("not using TLS", func() {
			err := tcpInput.Init(config)
			c.Assume(err, gs.IsNil)
			c.Expect(tcpInput.listener.Addr().String(), gs.Equals, ith.ResolvedAddrStr)

			c.Specify("accepts connections and passes them to the splitter", func() {
				go startServer()
				data := []byte("THIS IS THE DATA")

				outConn, err := net.Dial("tcp", ith.AddrStr)
				c.Assume(err, gs.IsNil)
				_, err = outConn.Write(data)
				c.Expect(err, gs.IsNil)
				outConn.Close()

				recd := <-bytesChan

				c.Expect(err, gs.IsNil)
				c.Expect(string(recd), gs.Equals, string(data))

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})
		})

		c.Specify("using TLS", func() {
			config.UseTls = true

			c.Specify("fails to init w/ missing key or cert file", func() {
				config.Tls = TlsConfig{}
				err := tcpInput.Init(config)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"TLS config requires both cert_file and key_file value.")
			})

			config.Tls = TlsConfig{
				CertFile: "./testsupport/cert.pem",
				KeyFile:  "./testsupport/key.pem",
			}

			c.Specify("accepts connections and passes them to the splitter", func() {
				err := tcpInput.Init(config)
				c.Expect(err, gs.IsNil)

				go startServer()
				data := []byte("This is a test.")

				clientConfig := new(tls.Config)
				clientConfig.InsecureSkipVerify = true
				outConn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
				c.Assume(err, gs.IsNil)
				outConn.SetWriteDeadline(time.Now().Add(time.Duration(10000)))
				n, err := outConn.Write(data)
				c.Expect(err, gs.IsNil)
				c.Expect(n, gs.Equals, len(data))
				outConn.Close()

				recd := <-bytesChan

				c.Expect(err, gs.IsNil)
				c.Expect(string(recd), gs.Equals, string(data))

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})

			c.Specify("doesn't accept connections below specified min TLS version", func() {
				config.Tls.MinVersion = "TLS12"
				err := tcpInput.Init(config)
				c.Expect(err, gs.IsNil)

				go startServer()

				clientConfig := &tls.Config{
					InsecureSkipVerify: true,
					MaxVersion:         tls.VersionTLS11,
				}
				conn, err := tls.Dial("tcp", ith.AddrStr, clientConfig)
				c.Expect(conn, gs.IsNil)
				c.Expect(err, gs.Not(gs.IsNil))

				<-bytesChan

				tcpInput.Stop()
				err = <-errChan
				c.Expect(err, gs.IsNil)
				srDoneWG.Wait()
			})
		})
	})
}