Example #1
0
func ProtobufDecoderSpec(c gospec.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	msg := ts.GetTestMessage()
	config := NewPipelineConfig(nil) // Initializes globals.

	c.Specify("A ProtobufDecoder", func() {
		encoded, err := proto.Marshal(msg)
		c.Assume(err, gs.IsNil)
		pack := NewPipelinePack(config.inputRecycleChan)
		decoder := new(ProtobufDecoder)
		decoder.sampleDenominator = 1000 // Since we don't call decoder.Init().

		c.Specify("decodes a protobuf message", func() {
			pack.MsgBytes = encoded
			_, err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(pack.Message, gs.Equals, msg)
			v, ok := pack.Message.GetFieldValue("foo")
			c.Expect(ok, gs.IsTrue)
			c.Expect(v, gs.Equals, "bar")
		})

		c.Specify("returns an error for bunk encoding", func() {
			bunk := append([]byte{0, 0, 0}, encoded...)
			pack.MsgBytes = bunk
			_, err := decoder.Decode(pack)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
func TestRun(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mockOR := pipelinemock.NewMockOutputRunner(mockCtrl)
	mockPH := pipelinemock.NewMockPluginHelper(mockCtrl)
	mockFirehose := NewMockRecordPutter(mockCtrl)

	firehoseOutput := FirehoseOutput{
		client: mockFirehose,
	}

	testChan := make(chan *pipeline.PipelinePack)
	mockOR.EXPECT().InChan().Return(testChan)

	// Send test input through the channel
	input := `{"key":"value"}`
	go func() {
		testPack := pipeline.PipelinePack{
			Message: &message.Message{
				Payload: &input,
			},
		}

		testChan <- &testPack
		close(testChan)
	}()

	mockFirehose.EXPECT().PutRecord([]byte(input)).Return(nil)

	err := firehoseOutput.Run(mockOR, mockPH)
	assert.NoError(t, err, "did not expect err for valid Run()")
}
Example #3
0
func StatsdInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())
	ith.PackSupply = make(chan *PipelinePack, 1)

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, input runner, and stat accumulator
	ith.MockHelper = NewMockPluginHelper(ctrl)
	ith.MockInputRunner = NewMockInputRunner(ctrl)
	mockStatAccum := NewMockStatAccumulator(ctrl)

	c.Specify("A StatsdInput", func() {
		statsdInput := StatsdInput{}
		config := statsdInput.ConfigStruct().(*StatsdInputConfig)

		config.Address = ith.AddrStr
		err := statsdInput.Init(config)
		c.Assume(err, gs.IsNil)
		realListener := statsdInput.listener
		c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr)
		realListener.Close()
		mockListener := pipeline_ts.NewMockConn(ctrl)
		statsdInput.listener = mockListener

		ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil)
		mockListener.EXPECT().Close()
		mockListener.EXPECT().SetReadDeadline(gomock.Any())

		c.Specify("sends a Stat to the StatAccumulator", func() {
			statName := "sample.count"
			statVal := 303
			msg := fmt.Sprintf("%s:%d|c\n", statName, statVal)
			expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)}
			mockStatAccum.EXPECT().DropStat(expected).Return(true)
			readCall := mockListener.EXPECT().Read(make([]byte, 512))
			readCall.Return(len(msg), nil)
			readCall.Do(func(msgBytes []byte) {
				copy(msgBytes, []byte(msg))
				statsdInput.Stop()
			})
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper)
				c.Expect(err, gs.IsNil)
				wg.Done()
			}()
			wg.Wait()
		})
	})
}
Example #4
0
func createFixtures(t *testing.T) (reporter *ErrorReporter, ctrl *gomock.Controller) {
	// reporter acts as a testing.T-like object that we pass to the
	// Controller. We use it to test that the mock considered tests
	// successful or failed.
	reporter = NewErrorReporter(t)
	ctrl = gomock.NewController(reporter)
	return
}
Example #5
0
func TestPanicOverridesExpectationChecks(t *testing.T) {
	ctrl := gomock.NewController(t)
	reporter := NewErrorReporter(t)

	reporter.assertFatal(func() {
		ctrl.RecordCall(new(Subject), "FooMethod", "1")
		defer ctrl.Finish()
		reporter.Fatalf("Intentional panic")
	})
}
Example #6
0
func TestCreateEndpoint(t *testing.T) {
	useMockFuncs()
	defer useStdFuncs()

	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckEndHandler := NewMockHandler(mockCtrl)

	Convey("Should allocate update endpoints", t, func() {
		app := NewApplication()
		app.endpointTemplate = testEndpointTemplate

		Convey("Should not encrypt endpoints without a key", func() {
			app.SetTokenKey("")
			app.SetEndpointHandler(mckEndHandler)

			mckEndHandler.EXPECT().URL().Return("https://example.com")

			endpoint, err := app.CreateEndpoint("123")
			So(err, ShouldBeNil)
			So(endpoint, ShouldEqual, "https://example.com/123")
		})

		Convey("Should encrypt endpoints with a key", func() {
			app.SetTokenKey("HVozKz_n-DPopP5W877DpRKQOW_dylVf")
			app.SetEndpointHandler(mckEndHandler)

			mckEndHandler.EXPECT().URL().Return("https://example.com")

			endpoint, err := app.CreateEndpoint("456")
			So(err, ShouldBeNil)
			So(endpoint, ShouldEqual,
				"https://example.com/AAAAAAAAAAAAAAAAAAAAAGMKig==")
		})

		Convey("Should reject invalid keys", func() {
			app.SetTokenKey("lLyhlLk8qus1ky4ER8yjN5o=")
			app.SetEndpointHandler(mckEndHandler)

			_, err := app.CreateEndpoint("123")
			So(err, ShouldEqual, aes.KeySizeError(17))
		})

		Convey("Should return a relative URL without an update handler", func() {
			app.SetTokenKey("O03rpLsdafhIhJEjEJt-CgVHyqHI650oy0pZZvplKDc=")
			endpoint, err := app.CreateEndpoint("789")
			So(err, ShouldBeNil)
			So(endpoint, ShouldEqual, "/AAAAAAAAAAAAAAAAAAAAAPfdsA==")
		})
	})
}
Example #7
0
func GeoIpDecoderSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := NewPipelineConfig(nil)
	pConfig.Globals.ShareDir = "/foo/bar/baz"

	c.Specify("A GeoIpDecoder", func() {
		decoder := new(GeoIpDecoder)
		decoder.SetPipelineConfig(pConfig)
		rec := new(geoip.GeoIPRecord)
		conf := decoder.ConfigStruct().(*GeoIpDecoderConfig)

		c.Expect(conf.DatabaseFile, gs.Equals, "/foo/bar/baz/GeoLiteCity.dat")

		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		nf, _ := message.NewField("remote_host", "74.125.142.147", "")
		pack.Message.AddField(nf)

		decoder.SourceIpField = "remote_host"
		conf.SourceIpField = "remote_host"
		decoder.Init(conf)

		rec.CountryCode = "US"
		rec.CountryCode3 = "USA"
		rec.CountryName = "United States"
		rec.Region = "CA"
		rec.City = "Mountain View"
		rec.PostalCode = "94043"
		rec.Latitude = 37.4192
		rec.Longitude = -122.0574
		rec.AreaCode = 650
		rec.CharSet = 1
		rec.ContinentCode = "NA"

		c.Specify("Test GeoIpDecoder Output", func() {
			buf := decoder.GeoBuff(rec)
			nf, _ = message.NewField("geoip", buf.Bytes(), "")
			pack.Message.AddField(nf)

			b, ok := pack.Message.GetFieldValue("geoip")
			c.Expect(ok, gs.IsTrue)

			c.Expect(string(b.([]byte)), gs.Equals, `{"latitude":37.4192008972168,"longitude":-122.0574035644531,"location":[-122.0574035644531,37.4192008972168],"coordinates":["-122.0574035644531","37.4192008972168"],"countrycode":"US","countrycode3":"USA","countryname":"United States","region":"CA","city":"Mountain View","postalcode":"94043","areacode":650,"charset":1,"continentcode":"NA"}`)
		})

	})
}
Example #8
0
func TestSocketListenConfig(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()
	mckStat := NewMockStatistician(mockCtrl)
	mckListenerConfig := NewMockListenerConfig(mockCtrl)

	app := NewApplication()
	app.hostname = "example.org"
	app.SetLogger(mckLogger)
	app.SetMetrics(mckStat)

	sh := NewSocketHandler()
	sh.setApp(app)

	// Should forward Listen errors.
	listenErr := errors.New("splines not reticulated")
	mckListenerConfig.EXPECT().Listen().Return(nil, listenErr)
	if err := sh.listenWithConfig(mckListenerConfig); err != listenErr {
		t.Errorf("Wrong error: got %#v; want %#v", err, listenErr)
	}

	// Should use the wss:// scheme if UseTLS returns true.
	ml := newMockListener(netAddr{"test", "[::1]:8080"})
	gomock.InOrder(
		mckListenerConfig.EXPECT().Listen().Return(ml, nil),
		mckListenerConfig.EXPECT().UseTLS().Return(true),
		mckListenerConfig.EXPECT().GetMaxConns().Return(1),
	)
	if err := sh.listenWithConfig(mckListenerConfig); err != nil {
		t.Errorf("Error setting listener: %s", err)
	}
	if maxConns := sh.MaxConns(); maxConns != 1 {
		t.Errorf("Mismatched maximum connection count: got %d; want 1",
			maxConns)
	}
	expectedURL := "wss://example.org:8080"
	if url := sh.URL(); url != expectedURL {
		t.Errorf("Mismatched handler URL: got %q; want %q",
			url, expectedURL)
	}
}
Example #9
0
// A more thorough test of notMatcher
func TestNotMatcher(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mockMatcher := mock_matcher.NewMockMatcher(ctrl)
	notMatcher := gomock.Not(mockMatcher)

	mockMatcher.EXPECT().Matches(4).Return(true)
	if match := notMatcher.Matches(4); match {
		t.Errorf("notMatcher should not match 4")
	}

	mockMatcher.EXPECT().Matches(5).Return(false)
	if match := notMatcher.Matches(5); !match {
		t.Errorf("notMatcher should match 5")
	}
}
// decodeMessageAndVerifyOutput takes a decoder conf, message payload, and a fn -> the fn is a number of
// assertions to verify that the message after decoding is as expected.
func decodeMessageAndVerifyOutput(c gs.Context, conf *JsonDecoderConfig, payload string, fn packVerifier) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// 1. Initialize test decoder
	decoder := new(JsonDecoder)
	err := decoder.Init(conf)
	c.Assume(err, gs.IsNil)
	dRunner := pipelinemock.NewMockDecoderRunner(ctrl)
	decoder.SetDecoderRunner(dRunner)

	// 2. Set payload to be tested, and decode it
	supply := make(chan *PipelinePack, 1)
	pack := NewPipelinePack(supply)
	pack.Message.SetPayload(payload)
	_, err = decoder.Decode(pack)

	// 3. Assert outcome of decoding
	fn(c, pack)
	pack.Zero()
}
Example #11
0
func BenchmarkCreateEndpoint(b *testing.B) {
	mockCtrl := gomock.NewController(b)
	defer mockCtrl.Finish()

	app := NewApplication()
	app.endpointTemplate = testEndpointTemplate
	app.SetTokenKey("")
	mckEndHandler := NewMockHandler(mockCtrl)
	mckEndHandler.EXPECT().URL().Return("https://example.com").Times(b.N)
	app.SetEndpointHandler(mckEndHandler)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		endpoint, err := app.CreateEndpoint("123")
		if err != nil {
			b.Fatalf("Error generating update endpoint: %s", err)
		}
		expected := "https://example.com/123"
		if endpoint != expected {
			b.Fatalf("Wrong endpoint: got %q; want %q", endpoint, expected)
		}
	}
}
Example #12
0
func InputRunnerSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	globals := &GlobalConfigStruct{
		PluginChanSize: 5,
	}
	pc := NewPipelineConfig(globals)

	mockHelper := NewMockPluginHelper(ctrl)

	c.Specify("Runner restarts a plugin on the first time only", func() {
		var pluginGlobals PluginGlobals
		pluginGlobals.Retries = RetryOptions{
			MaxDelay:   "1us",
			Delay:      "1us",
			MaxJitter:  "1us",
			MaxRetries: 1,
		}

		pw := NewPluginWrapper("stopping", pc)
		pw.ConfigCreator = func() interface{} { return nil }
		pw.PluginCreator = func() interface{} { return new(StoppingInput) }
		pc.inputWrappers["stopping"] = pw

		input := new(StoppingInput)
		iRunner := NewInputRunner("stopping", input, &pluginGlobals, false)
		var wg sync.WaitGroup
		cfgCall := mockHelper.EXPECT().PipelineConfig().Times(2)
		cfgCall.Return(pc)
		wg.Add(1)
		iRunner.Start(mockHelper, &wg)
		wg.Wait()
		c.Expect(stopinputTimes, gs.Equals, 2)
	})
}
// TestRunWithTimestamp tests that if a TimestampColumn is provided in the config
// then the Heka message's timestamp gets added to the message with that column name.
func TestRunWithTimestamp(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mockOR := pipelinemock.NewMockOutputRunner(mockCtrl)
	mockPH := pipelinemock.NewMockPluginHelper(mockCtrl)
	mockFirehose := NewMockRecordPutter(mockCtrl)

	firehoseOutput := FirehoseOutput{
		client:          mockFirehose,
		timestampColumn: "created",
	}

	testChan := make(chan *pipeline.PipelinePack)
	mockOR.EXPECT().InChan().Return(testChan)

	// Send test input through the channel
	input := `{}`
	timestamp := time.Date(2015, 07, 1, 13, 14, 15, 0, time.UTC).UnixNano()
	go func() {
		testPack := pipeline.PipelinePack{
			Message: &message.Message{
				Payload:   &input,
				Timestamp: &timestamp,
			},
		}

		testChan <- &testPack
		close(testChan)
	}()

	expected := `{"created":"2015-07-01 13:14:15.000"}`
	mockFirehose.EXPECT().PutRecord([]byte(expected)).Return(nil)

	err := firehoseOutput.Run(mockOR, mockPH)
	assert.NoError(t, err, "did not expect err for valid Run()")
}
Example #14
0
func ScribbleDecoderSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	c.Specify("A ScribbleDecoder", func() {
		decoder := new(ScribbleDecoder)
		config := decoder.ConfigStruct().(*ScribbleDecoderConfig)
		myType := "myType"
		myPayload := "myPayload"
		config.MessageFields = MessageTemplate{"Type": myType, "Payload": myPayload}
		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		c.Specify("sets basic values correctly", func() {
			decoder.Init(config)
			packs, err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(len(packs), gs.Equals, 1)
			c.Expect(pack.Message.GetType(), gs.Equals, myType)
			c.Expect(pack.Message.GetPayload(), gs.Equals, myPayload)
		})
	})
}
Example #15
0
func TestLocatorReadyNotify(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	uaid := "fce61180716a40ed8e79bf5ff0ba34bc"

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(), gomock.Any(),
		gomock.Any()).AnyTimes()

	var (
		// routerPipes maps fake peer addresses to their respective pipes. Used
		// by dialRouter to connect to peers.
		routerPipes = make(map[netAddr]*pipeListener)

		// contacts is a list of peer URLs for the locator.
		contacts []string
	)

	// Fake listener for the sender's router, used to test self-routing.
	sndRouterAddr := netAddr{"tcp", "snd-router.example.com:3000"}
	sndRouterPipe := newPipeListener()
	defer sndRouterPipe.Close()
	routerPipes[sndRouterAddr] = sndRouterPipe
	contacts = append(contacts, "http://snd-router.example.com:3000")

	// Fake listener for the receiver's router, used to test routing updates
	// to different hosts.
	recvRouterAddr := netAddr{"tcp", "recv-router.example.com:3000"}
	recvRouterPipe := newPipeListener()
	defer recvRouterPipe.Close()
	routerPipes[recvRouterAddr] = recvRouterPipe
	contacts = append(contacts, "http://recv-router.example.com:3000")

	// Fake listener for the receiver's WebSocket handler, used to accept a
	// WebSocket client connection.
	socketHandlerPipe := newPipeListener()
	defer socketHandlerPipe.Close()

	// Fake locator.
	mckLocator := NewMockLocator(mockCtrl)
	mckLocator.EXPECT().Contacts(uaid).Return(contacts, nil).Times(2)

	// Fake dialer to connect to each peer's routing listener.
	dialRouter := func(network, address string) (net.Conn, error) {
		if pipe, ok := routerPipes[netAddr{network, address}]; ok {
			return pipe.Dial(network, address)
		}
		return nil, &netErr{temporary: false, timeout: false}
	}
	// Configures a fake router for the app.
	setRouter := func(app *Application, listener net.Listener) {
		r := NewBroadcastRouter()
		r.setApp(app)
		r.setClientOptions(10, 3*time.Second, 3*time.Second) // Defaults.
		r.setClientTransport(&http.Transport{Dial: dialRouter})
		r.listenWithConfig(listenerConfig{listener: listener})
		r.maxDataLen = 4096
		r.server = newServeWaiter(&http.Server{Handler: r.ServeMux()})
		app.SetRouter(r)
	}

	// sndApp is the server broadcasting the update. The locator returns the
	// addresses of the sender and receiver to test self-routing.
	sndApp := NewApplication()
	sndApp.SetLogger(mckLogger)
	sndStat := NewMockStatistician(mockCtrl)
	sndApp.SetMetrics(sndStat)
	sndStore := NewMockStore(mockCtrl)
	sndApp.SetStore(sndStore)
	sndApp.SetLocator(mckLocator)
	// Set up a fake router for the sender.
	setRouter(sndApp, sndRouterPipe)

	// recvApp is the server receiving the update.
	recvApp := NewApplication()
	recvApp.SetLogger(mckLogger)
	recvStat := NewMockStatistician(mockCtrl)
	recvApp.SetMetrics(recvStat)
	recvStore := NewMockStore(mockCtrl)
	recvApp.SetStore(recvStore)
	// Wrap the fake locator in a type that implements ReadyNotifier.
	recvLocator := newMockReadyNotifier(mckLocator)
	recvApp.SetLocator(recvLocator)
	// Set up a fake WebSocket handler for the receiver.
	recvSocketHandler := NewSocketHandler()
	recvSocketHandler.setApp(recvApp)
	recvSocketHandler.listenWithConfig(listenerConfig{
		listener: socketHandlerPipe})
	recvSocketHandler.server = newServeWaiter(&http.Server{Handler: recvSocketHandler.ServeMux()})
	recvApp.SetSocketHandler(recvSocketHandler)
	// Set up a fake router for the receiver.
	setRouter(recvApp, recvRouterPipe)

	chid := "2b7c5c27d6224bfeaf1c158c3c57fca3"
	version := int64(2)
	data := "I'm a little teapot, short and stout."

	var wg sync.WaitGroup // Waits for the client to close.
	wg.Add(1)
	dialChan := make(chan bool) // Signals when the client connects.
	timeout := closeAfter(2 * time.Second)

	go func() {
		defer wg.Done()
		origin := &url.URL{Scheme: "ws", Host: "recv-conn.example.com"}
		ws, err := dialSocketListener(socketHandlerPipe, &websocket.Config{
			Location: origin,
			Origin:   origin,
			Version:  websocket.ProtocolVersionHybi13,
		})
		if err != nil {
			t.Errorf("Error dialing host: %s", err)
			return
		}
		defer ws.Close()
		err = websocket.JSON.Send(ws, struct {
			Type       string   `json:"messageType"`
			DeviceID   string   `json:"uaid"`
			ChannelIDs []string `json:"channelIDs"`
		}{"hello", uaid, []string{}})
		if err != nil {
			t.Errorf("Error writing handshake request: %s", err)
			return
		}
		helloReply := new(HelloReply)
		if err = websocket.JSON.Receive(ws, helloReply); err != nil {
			t.Errorf("Error reading handshake reply: %s", err)
			return
		}
		select {
		case dialChan <- true:
		case <-timeout:
			t.Errorf("Timed out waiting for router")
			return
		}
		flushReply := new(FlushReply)
		if err = websocket.JSON.Receive(ws, flushReply); err != nil {
			t.Errorf("Error reading routed update: %s", err)
			return
		}
		ok := false
		expected := Update{chid, uint64(version), data}
		for _, update := range flushReply.Updates {
			if ok = update == expected; ok {
				break
			}
		}
		if !ok {
			t.Errorf("Missing update %#v in %#v", expected, flushReply.Updates)
			return
		}
	}()

	// Start the handlers.
	errChan := make(chan error, 3)
	go sndApp.Router().Start(errChan)
	go recvApp.SocketHandler().Start(errChan)
	go recvApp.Router().Start(errChan)

	// First and second routing attempts to self.
	sndStat.EXPECT().Increment("updates.routed.unknown").Times(2)

	// Initial routing attempt to peer.
	recvStat.EXPECT().Increment("updates.routed.unknown")
	// Client connects to peer.
	recvStat.EXPECT().Increment("client.socket.connect")
	recvStore.EXPECT().CanStore(0).Return(true)
	recvStat.EXPECT().Increment("updates.client.hello")
	recvStore.EXPECT().FetchAll(uaid, gomock.Any()).Return(nil, nil, nil)
	recvStat.EXPECT().Timer("client.flush", gomock.Any())
	// Second routing attempt to peer.
	recvStat.EXPECT().Increment("updates.routed.incoming")
	recvStat.EXPECT().Increment("updates.sent")
	recvStat.EXPECT().Timer("client.flush", gomock.Any())
	recvStat.EXPECT().Increment("updates.routed.received")
	recvStat.EXPECT().Timer("client.socket.lifespan", gomock.Any())
	recvStat.EXPECT().Increment("client.socket.disconnect")

	// Initial routing attempt should fail; the WebSocket listener shouldn't
	// accept client connections before the locator is ready.
	delivered, err := sndApp.Router().Route(nil, uaid, chid, version, timeNow(),
		"disconnected", data)
	if err != nil {
		t.Errorf("Error routing to disconnected client: %s", err)
	} else if delivered {
		t.Error("Should not route to disconnected client")
	}
	// Signal the locator is ready, then wait for the client to connect.
	recvLocator.SignalReady()
	select {
	case <-dialChan:
	case <-time.After(5 * time.Second):
		t.Fatalf("Timed out waiting for the client to connect")
	}
	// Routing should succeed once the client is connected.
	delivered, err = sndApp.Router().Route(nil, uaid, chid, version, timeNow(),
		"connected", data)
	if err != nil {
		t.Errorf("Error routing to connected client: %s", err)
	} else if !delivered {
		t.Error("Should route to connected client")
	}

	gomock.InOrder(
		mckLocator.EXPECT().Close(),
		recvStore.EXPECT().Close(),
	)
	if err := recvApp.Close(); err != nil {
		t.Errorf("Error closing peer: %s", err)
	}
	wg.Wait()
	gomock.InOrder(
		mckLocator.EXPECT().Close(),
		sndStore.EXPECT().Close(),
	)
	if err := sndApp.Close(); err != nil {
		t.Errorf("Error closing self: %s", err)
	}
	// Wait for the handlers to stop.
	for i := 0; i < 3; i++ {
		<-errChan
	}
}
Example #16
0
func ReportSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	chanSize := pConfig.Globals.PluginChanSize

	checkForFields := func(c gs.Context, msg *message.Message) {
		f0Val, ok := msg.GetFieldValue(f0.GetName())
		c.Expect(ok, gs.IsTrue)
		c.Expect(f0Val.(int64), gs.Equals, f0.GetValue().(int64))
		f1Val, ok := msg.GetFieldValue(f1.GetName())
		c.Expect(ok, gs.IsTrue)
		c.Expect(f1Val.(string), gs.Equals, f1.GetValue().(string))
	}

	hasChannelData := func(msg *message.Message) (ok bool) {
		capVal, _ := msg.GetFieldValue("InChanCapacity")
		lenVal, _ := msg.GetFieldValue("InChanLength")
		var i int64
		if i, ok = capVal.(int64); !ok {
			return
		}
		if ok = (i == int64(chanSize)); !ok {
			return
		}
		if i, ok = lenVal.(int64); !ok {
			return
		}
		ok = (i == int64(0))
		return
	}

	fName := "counter"
	filter := new(CounterFilter)
	fRunner := NewFORunner(fName, filter, nil, chanSize)
	var err error
	fRunner.matcher, err = NewMatchRunner("Type == ''", "", fRunner, chanSize)
	c.Assume(err, gs.IsNil)
	fRunner.matcher.inChan = make(chan *PipelinePack, chanSize)
	leakCount := 10
	fRunner.SetLeakCount(leakCount)

	iName := "stat_accum"
	input := new(StatAccumInput)
	iRunner := NewInputRunner(iName, input, nil, false)

	c.Specify("`PopulateReportMsg`", func() {
		msg := ts.GetTestMessage()

		c.Specify("w/ a filter", func() {
			err := PopulateReportMsg(fRunner, msg)
			c.Assume(err, gs.IsNil)

			c.Specify("invokes `ReportMsg` on the filter", func() {
				checkForFields(c, msg)
			})

			c.Specify("adds the channel data", func() {
				c.Expect(hasChannelData(msg), gs.IsTrue)
			})

			c.Specify("has its leak count set properly", func() {
				leakVal, ok := msg.GetFieldValue("LeakCount")
				c.Assume(ok, gs.IsTrue)
				i, ok := leakVal.(int64)
				c.Assume(ok, gs.IsTrue)
				c.Expect(int(i), gs.Equals, leakCount)
			})
		})

		c.Specify("w/ an input", func() {
			err := PopulateReportMsg(iRunner, msg)
			c.Assume(err, gs.IsNil)

			c.Specify("invokes `ReportMsg` on the input", func() {
				checkForFields(c, msg)
			})

			c.Specify("doesn't add any channel data", func() {
				capVal, ok := msg.GetFieldValue("InChanCapacity")
				c.Expect(capVal, gs.IsNil)
				c.Expect(ok, gs.IsFalse)
				lenVal, ok := msg.GetFieldValue("InChanLength")
				c.Expect(lenVal, gs.IsNil)
				c.Expect(ok, gs.IsFalse)
			})
		})
	})

	c.Specify("PipelineConfig", func() {
		pc := NewPipelineConfig(nil)
		// Initialize all of the PipelinePacks that we'll need
		pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)

		pc.FilterRunners = map[string]FilterRunner{fName: fRunner}
		pc.InputRunners = map[string]InputRunner{iName: iRunner}

		c.Specify("returns full set of accurate reports", func() {
			reportChan := make(chan *PipelinePack)
			go pc.reports(reportChan)

			reports := make(map[string]*PipelinePack)
			for r := range reportChan {
				iName, ok := r.Message.GetFieldValue("name")
				c.Expect(ok, gs.IsTrue)
				name, ok := iName.(string)
				c.Expect(ok, gs.IsTrue)
				c.Expect(name, gs.Not(gs.Equals), "MISSING")
				reports[name] = r
				pc.reportRecycleChan <- NewPipelinePack(pc.reportRecycleChan)
			}
			fReport := reports[fName]
			c.Expect(fReport, gs.Not(gs.IsNil))
			checkForFields(c, fReport.Message)
			c.Expect(hasChannelData(fReport.Message), gs.IsTrue)

			iReport := reports[iName]
			c.Expect(iReport, gs.Not(gs.IsNil))
			checkForFields(c, iReport.Message)

			recycleReport := reports["inputRecycleChan"]
			c.Expect(recycleReport, gs.Not(gs.IsNil))
			capVal, ok := recycleReport.Message.GetFieldValue("InChanCapacity")
			c.Expect(ok, gs.IsTrue)
			c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))

			injectReport := reports["injectRecycleChan"]
			c.Expect(injectReport, gs.Not(gs.IsNil))
			capVal, ok = injectReport.Message.GetFieldValue("InChanCapacity")
			c.Expect(ok, gs.IsTrue)
			c.Expect(capVal.(int64), gs.Equals, int64(pConfig.Globals.PoolSize))

			routerReport := reports["Router"]
			c.Expect(routerReport, gs.Not(gs.IsNil))
			c.Expect(hasChannelData(routerReport.Message), gs.IsTrue)
		})
	})
}
func KeenOutputSpec(c gs.Context) {
	ctrl := gomock.NewController(universalT)
	defer ctrl.Finish()

	c.Specify("A KeenOutput", func() {

		successTests := []SuccessfulTestCase{
			SuccessfulTestCase{
				"successfully records a valid job-finished message",
				"{\"JobType\":\"nyc_aris\",\"SystemId\":\"1234567890abcdefghijklmn\",\"TimeCreated\":\"2014-07-03T23:35:24.000Z\",\"Duration\":38900,\"Success\":true,\"Message\":\"\"}",
				func(eventData map[string]interface{}) bool {
					return eventData["JobType"] == "nyc_aris" &&
						eventData["SystemId"] == "1234567890abcdefghijklmn" &&
						eventData["TimeCreated"] == "2014-07-03T23:35:24.000Z" &&
						eventData["Duration"] == float64(38900) &&
						eventData["Success"] == true &&
						eventData["Message"] == ""
				},
			},
		}

		errorTests := []ErrorTestCase{
			ErrorTestCase{
				"logs an error but does not crash when the message payload is not valid JSON",
				"not json",
				"*json.SyntaxError",
				"invalid character 'o' in literal null (expecting 'u')",
			},
		}

		for _, test := range successTests {
			oth := NewOutputTestHelper(universalT, ctrl)
			output := new(KeenOutput)
			output.Init(&KeenOutputConfig{Collection: "job-finished"})
			mockClient := MockKeenClient{mock.Mock{}}
			output.client = &mockClient

			inChan := make(chan *pipeline.PipelinePack, 1)
			oth.MockOutputRunner.EXPECT().On("InChan").Return(inChan)
			mockClient.EXPECT().On("AddEvent", "job-finished", mock.Anything).Return(nil)

			pack := getEmptyKeenOutputPack()
			pack.Message.SetPayload(test.MessagePayload)
			inChan <- pack
			close(inChan)

			output.Run(oth.MockOutputRunner, oth.MockHelper)

			ExpectCall(universalT, &mockClient.mock, "AddEvent with expected JSON", "AddEvent",
				func(args []interface{}) bool {
					if len(args) != 2 {
						return false
					}
					eventData, ok := args[1].(map[string]interface{})
					return ok && test.IsEventDataCorrect(eventData)
				})
		}

		for _, test := range errorTests {
			oth := NewOutputTestHelper(universalT, ctrl)
			output := new(KeenOutput)
			output.Init(&KeenOutputConfig{})
			mockClient := MockKeenClient{mock.Mock{}}
			output.client = &mockClient

			inChan := make(chan *pipeline.PipelinePack, 1)
			oth.MockOutputRunner.EXPECT().On("InChan").Return(inChan)
			oth.MockOutputRunner.EXPECT().On("LogError", mock.AnythingOfType(test.ExpectedErrorType)).Return()

			pack := getEmptyKeenOutputPack()
			pack.Message.SetPayload(test.MessagePayload)
			inChan <- pack
			close(inChan)

			output.Run(oth.MockOutputRunner, oth.MockHelper)

			ExpectCall(universalT, &oth.MockOutputRunner.mock, "Log correct error", "LogError",
				func(args []interface{}) bool {
					if len(args) != 1 {
						return false
					}
					err, ok := args[0].(error)
					return ok && err.Error() == test.ExpectedErrorMessage
				})

			oth.MockOutputRunner.EXPECT().AssertExpectations(universalT)
		}
	})
}
Example #18
0
func HttpInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)

	json_post := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`

	c.Specify("A HttpInput", func() {

		httpInput := HttpInput{}
		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
		ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

		runOutputChan := make(chan error, 1)
		startInput := func() {
			go func() {
				runOutputChan <- httpInput.Run(ith.MockInputRunner, ith.MockHelper)
			}()
		}

		ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

		// These assume that every sub-spec starts the input.
		config := httpInput.ConfigStruct().(*HttpInputConfig)
		tickChan := make(chan time.Time)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
		ith.MockHelper.EXPECT().Hostname().Return("hekatests.example.com")

		// These assume that every sub-spec makes exactly one HTTP request.
		ith.MockInputRunner.EXPECT().NewSplitterRunner("0").Return(ith.MockSplitterRunner)
		getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
		getRecCall.Return(len(json_post), []byte(json_post), io.EOF)
		ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
		decChan := make(chan func(*PipelinePack), 1)
		packDecCall := ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
		packDecCall.Do(func(dec func(*PipelinePack)) {
			decChan <- dec
		})
		ith.MockSplitterRunner.EXPECT().DeliverRecord([]byte(json_post), nil)
		ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false).AnyTimes()
		splitter := &TokenSplitter{} // not actually used
		ith.MockSplitterRunner.EXPECT().Splitter().Return(splitter)

		c.Specify("honors time ticker to flush", func() {
			// Spin up a http server.
			server, err := plugins_ts.NewOneHttpServer(json_post, "localhost", 9876)
			c.Expect(err, gs.IsNil)
			go server.Start("/")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9876/"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			// Getting the decorator means we've made our HTTP request.
			<-decChan
		})

		c.Specify("supports configuring HTTP Basic Authentication", func() {
			// Spin up a http server which expects username "user" and password "password"
			server, err := plugins_ts.NewHttpBasicAuthServer("user", "password", "localhost", 9875)
			c.Expect(err, gs.IsNil)
			go server.Start("/BasicAuthTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9875/BasicAuthTest"
			config.User = "******"
			config.Password = "******"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring a different HTTP method", func() {
			// Spin up a http server which expects requests with method "POST"
			server, err := plugins_ts.NewHttpMethodServer("POST", "localhost", 9874)
			c.Expect(err, gs.IsNil)
			go server.Start("/PostTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9874/PostTest"
			config.Method = "POST"

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring HTTP headers", func() {
			// Spin up a http server which expects requests with method "POST"
			server, err := plugins_ts.NewHttpHeadersServer(map[string]string{"Accept": "text/plain"}, "localhost", 9873)
			c.Expect(err, gs.IsNil)
			go server.Start("/HeadersTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9873/HeadersTest"
			config.Headers = map[string]string{"Accept": "text/plain"}

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()
			tickChan <- time.Now()

			dec := <-decChan
			dec(ith.Pack)

			// we expect a statuscode 200 (i.e. success)
			statusCode, ok := ith.Pack.Message.GetFieldValue("StatusCode")
			c.Assume(ok, gs.IsTrue)
			c.Expect(statusCode, gs.Equals, int64(200))
		})

		c.Specify("supports configuring a request body", func() {
			// Spin up a http server that echoes back the request body
			server, err := plugins_ts.NewHttpBodyServer("localhost", 9872)
			c.Expect(err, gs.IsNil)
			go server.Start("/BodyTest")
			time.Sleep(10 * time.Millisecond)

			config.Url = "http://localhost:9872/BodyTest"
			config.Method = "POST"
			config.Body = json_post

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			respBodyChan := make(chan []byte, 1)
			getRecCall.Do(func(r io.Reader) {
				respBody := make([]byte, len(config.Body))
				n, err := r.Read(respBody)
				c.Expect(n, gs.Equals, len(config.Body))
				c.Expect(err, gs.Equals, io.EOF)
				respBodyChan <- respBody
			})

			startInput()
			tickChan <- time.Now()

			respBody := <-respBodyChan
			c.Expect(string(respBody), gs.Equals, json_post)
		})

		httpInput.Stop()
		runOutput := <-runOutputChan
		c.Expect(runOutput, gs.IsNil)
	})
}
Example #19
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	config := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(config.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.Decoder = pipelinemock.NewMockDecoderRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)
	ith.DecodeChan = make(chan *PipelinePack)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: config}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}
		lsiConfig.Decoder = "decoder-name"

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)
			mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply).Times(numLines)
				// Expect calls to get decoder and decode each message. Since the
				// decoding is a no-op, the message payload will be the log file
				// line, unchanged.
				pbcall := ith.MockHelper.EXPECT().DecoderRunner(lsiConfig.Decoder,
					"-"+lsiConfig.Decoder)
				pbcall.Return(mockDecoderRunner, true)

				decodeCall := mockDecoderRunner.EXPECT().InChan().Times(numLines)
				decodeCall.Return(ith.DecodeChan)

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				d, _ := time.ParseDuration("5s")
				timeout := time.After(d)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case <-ith.DecodeChan:
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Example #20
0
func LogstreamerInputSpec(c gs.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	here, _ := os.Getwd()
	dirPath := filepath.Join(here, "../../logstreamer", "testdir", "filehandling/subdir")

	tmpDir, tmpErr := ioutil.TempDir("", "hekad-tests")
	c.Expect(tmpErr, gs.Equals, nil)
	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.IsNil)
	}()

	globals := DefaultGlobals()
	globals.BaseDir = tmpDir
	pConfig := NewPipelineConfig(globals)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// Specify localhost, but we're not really going to use the network.
	ith.AddrStr = "localhost:55565"
	ith.ResolvedAddrStr = "127.0.0.1:55565"

	// Set up mock helper, runner, and pack supply channel.
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	c.Specify("A LogstreamerInput", func() {
		lsInput := &LogstreamerInput{pConfig: pConfig}
		lsiConfig := lsInput.ConfigStruct().(*LogstreamerInputConfig)
		lsiConfig.LogDirectory = dirPath
		lsiConfig.FileMatch = `file.log(\.?)(?P<Seq>\d+)?`
		lsiConfig.Differentiator = []string{"logfile"}
		lsiConfig.Priority = []string{"^Seq"}

		c.Specify("w/ no translation map", func() {
			err := lsInput.Init(lsiConfig)
			c.Expect(err, gs.IsNil)
			c.Expect(len(lsInput.plugins), gs.Equals, 1)

			// Create pool of packs.
			numLines := 5 // # of lines in the log file we're parsing.
			packs := make([]*PipelinePack, numLines)
			ith.PackSupply = make(chan *PipelinePack, numLines)
			for i := 0; i < numLines; i++ {
				packs[i] = NewPipelinePack(ith.PackSupply)
				ith.PackSupply <- packs[i]
			}

			c.Specify("reads a log file", func() {
				// Expect InputRunner calls to get InChan and inject outgoing msgs.
				ith.MockInputRunner.EXPECT().LogError(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).AnyTimes()
				ith.MockInputRunner.EXPECT().NewDeliverer("1").Return(ith.MockDeliverer)
				ith.MockInputRunner.EXPECT().NewSplitterRunner("1").Return(
					ith.MockSplitterRunner)
				ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
				ith.MockSplitterRunner.EXPECT().IncompleteFinal().Return(false)
				ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())

				getRecCall := ith.MockSplitterRunner.EXPECT().GetRecordFromStream(
					gomock.Any()).Times(numLines)
				line := "boo hoo foo foo"
				getRecCall.Return(len(line), []byte(line), nil)
				getRecCall = ith.MockSplitterRunner.EXPECT().GetRecordFromStream(gomock.Any())
				getRecCall.Return(0, make([]byte, 0), io.EOF)

				deliverChan := make(chan []byte, 1)
				deliverCall := ith.MockSplitterRunner.EXPECT().DeliverRecord(gomock.Any(),
					ith.MockDeliverer).Times(numLines)
				deliverCall.Do(func(record []byte, del Deliverer) {
					deliverChan <- record
				})

				ith.MockDeliverer.EXPECT().Done()

				runOutChan := make(chan error, 1)
				go func() {
					err = lsInput.Run(ith.MockInputRunner, ith.MockHelper)
					runOutChan <- err
				}()

				dur, _ := time.ParseDuration("5s")
				timeout := time.After(dur)
				timed := false
				for x := 0; x < numLines; x++ {
					select {
					case record := <-deliverChan:
						c.Expect(string(record), gs.Equals, line)
					case <-timeout:
						timed = true
						x += numLines
					}
					// Free up the scheduler while we wait for the log file lines
					// to be processed.
					runtime.Gosched()
				}
				lsInput.Stop()
				c.Expect(timed, gs.Equals, false)
				c.Expect(<-runOutChan, gs.Equals, nil)
			})
		})

		c.Specify("with a translation map", func() {
			lsiConfig.Translation = make(ls.SubmatchTranslationMap)
			lsiConfig.Translation["Seq"] = make(ls.MatchTranslationMap)

			c.Specify("allows len 1 translation map for 'missing'", func() {
				lsiConfig.Translation["Seq"]["missing"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.IsNil)
			})

			c.Specify("doesn't allow len 1 map for other keys", func() {
				lsiConfig.Translation["Seq"]["missin"] = 9999
				err := lsInput.Init(lsiConfig)
				c.Expect(err, gs.Not(gs.IsNil))
				c.Expect(err.Error(), gs.Equals,
					"A translation map with one entry ('Seq') must be specifying a "+
						"'missing' key.")
			})
		})
	})
}
Example #21
0
func TestEndpointResolveKey(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()

	mckStat := NewMockStatistician(mockCtrl)
	mckStore := NewMockStore(mockCtrl)

	Convey("Endpoint tokens", t, func() {
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)
		app.SetStore(mckStore)

		Convey("Should return a 404 for invalid tokens", func() {
			app.SetTokenKey("c3v0AlmmxXu_LSfdZY3l3eayLsIwkX48")
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL: &url.URL{
					Path: "/update/j1bqzFq9WiwFZbqay-y7xVlfSvtO1eY="}, // "123.456"
			}
			gomock.InOrder(
				mckStore.EXPECT().KeyToIDs("123.456").Return("", "", ErrInvalidKey),
				mckStat.EXPECT().Increment("updates.appserver.invalid"),
			)
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 404)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Invalid Token"`)
		})

		Convey("Should not decode plaintext tokens without a key", func() {
			var err error

			app.SetTokenKey("")
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			_, err = eh.decodePK("")
			So(err, ShouldNotBeNil)

			pk, err := eh.decodePK("123.456")
			So(pk, ShouldEqual, "123.456")
		})

		Convey("Should normalize decoded tokens", func() {
			app.SetTokenKey("LM1xDImCx0rB46LCnx-3v4-Iyfk1LeKJbx9wuvx_z3U=")
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			// Hyphenated IDs should be normalized.
			uaid := "dbda2ba2-004c-491f-9e3d-c5950aee93de"
			chid := "848cd568-3f2a-4108-9ce4-bd0d928ecad4"
			// " \t%s.%s\r\n" % (uaid, chid)
			encodedKey := "qfGSdZzwf20GXiYubmZfIXj11Rx4RGJujFsjSQGdF4LRBhHbB_vt3hdW7cRvL9Fq_t_guMBGkDgebOoa5gRd1GGLN-Cv6h5hkpRTbdju8Tk-hMyC91BP4CEres_8"

			// decodePK should trim whitespace from encoded keys.
			mckStore.EXPECT().KeyToIDs(
				fmt.Sprintf("%s.%s", uaid, chid)).Return(uaid, chid, nil)
			actualUAID, actualCHID, err := eh.resolvePK(encodedKey)
			So(err, ShouldBeNil)
			So(actualUAID, ShouldEqual, uaid)
			So(actualCHID, ShouldEqual, chid)
		})

		Convey("Should reject invalid tokens", func() {
			var err error

			app.SetTokenKey("IhnNwMNbsFWiafTXSgF4Ag==")
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			invalidKey := "b54QOw2omSWBiEq0IuyfBGxHBIR7AI9YhCMA0lP9" // "_=!@#$%^&*()[]"
			uaid := "82398a648c834f8b838cb3945eceaf29"
			chid := "af445ad07e5f46b7a6c858150fc5aa92"
			validKey := fmt.Sprintf("%s.%s", uaid, chid)
			encodedKey := "swKSH8P2qprRt5y0J4Wi7ybl-qzFv1j09WPOfuabpEJmVUqwUpxjprXc2R3Yw0ITbqc_Swntw9_EpCgo_XuRTn7Q7opQYoQUgMPhCgT0EGbK"

			_, _, err = eh.resolvePK(invalidKey[:8])
			So(err, ShouldNotBeNil)

			_, _, err = eh.resolvePK(invalidKey)
			So(err, ShouldNotBeNil)

			// Reject plaintext tokens if a key is specified.
			_, _, err = eh.resolvePK(validKey)
			So(err, ShouldNotBeNil)

			mckStore.EXPECT().KeyToIDs(validKey).Return("", "", ErrInvalidKey)
			_, _, err = eh.resolvePK(encodedKey)
			So(err, ShouldNotBeNil)

			mckStore.EXPECT().KeyToIDs(validKey).Return(uaid, chid, nil)
			actualUAID, actualCHID, err := eh.resolvePK(encodedKey)
			So(err, ShouldBeNil)
			So(actualUAID, ShouldEqual, uaid)
			So(actualCHID, ShouldEqual, chid)
		})
	})
}
Example #22
0
func TestEndpointDelivery(t *testing.T) {
	useMockFuncs()
	defer useStdFuncs()

	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()
	mckStat := NewMockStatistician(mockCtrl)
	mckStore := NewMockStore(mockCtrl)
	mckRouter := NewMockRouter(mockCtrl)
	mckWorker := NewMockWorker(mockCtrl)

	Convey("Update delivery", t, func() {
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)
		app.SetStore(mckStore)
		app.SetRouter(mckRouter)

		Convey("Should attempt local delivery if `AlwaysRoute` is disabled", func() {
			eh := NewEndpointHandler()
			eh.setApp(app)
			app.SetEndpointHandler(eh)

			Convey("Should route updates if the device is not connected", func() {
				uaid := "f7e9fc483f7344c398701b6fa0e85e4f"
				chid := "737b7a0d25674be4bb184f015fce02cf"
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, int64(3), timeNow().UTC(),
						"", "").Return(true, nil),
					mckStat.EXPECT().Increment("router.broadcast.hit"),
					mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
					mckStat.EXPECT().Increment("updates.appserver.received"),
				)
				ok := eh.deliver(nil, uaid, chid, 3, "", "")
				So(ok, ShouldBeTrue)
			})

			// A routing failure still stores the alert for potential
			// client delivery. We should only return 404 for absolute
			// failures (where the endpoint is no longer valid
			Convey("Should return a 202 if routing fails", func() {
				resp := httptest.NewRecorder()
				req := &http.Request{
					Method: "PUT",
					Header: http.Header{HeaderID: {"reqID"}},
					URL:    &url.URL{Path: "/update/123"},
					Body:   formReader(url.Values{"version": {"1"}}),
				}
				gomock.InOrder(
					mckStore.EXPECT().KeyToIDs("123").Return("123", "456", nil),
					mckStat.EXPECT().Increment("updates.appserver.incoming"),
					mckStore.EXPECT().Update("123", "456", int64(1)).Return(nil),
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, "123", "456", int64(1),
						gomock.Any(), "reqID", "").Return(false, nil),
					mckStat.EXPECT().Increment("router.broadcast.miss"),
					mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
					mckStat.EXPECT().Increment("updates.appserver.rejected"),
				)
				eh.ServeMux().ServeHTTP(resp, req)

				So(resp.Code, ShouldEqual, 202)
				body, isJSON := getJSON(resp.HeaderMap, resp.Body)
				So(isJSON, ShouldBeTrue)
				So(body.String(), ShouldEqual, "{}")
			})

			Convey("Should return a 404 if local delivery fails", func() {
				uaid := "9e98d6415d8e4fd099ab1bad7178f750"
				chid := "0eecf572e99f4d508666d8da6c0b15a9"
				app.AddWorker(uaid, mckWorker)

				gomock.InOrder(
					mckWorker.EXPECT().Send(chid, int64(3), "").Return(
						errors.New("client gone")),
					mckStat.EXPECT().Increment("updates.appserver.rejected"),
				)
				ok := eh.deliver(nil, uaid, chid, int64(3), "", "")
				So(ok, ShouldBeFalse)
			})

			Convey("Should return an error if storage is unavailable", func() {
				resp := httptest.NewRecorder()
				req := &http.Request{
					Method: "PUT",
					Header: http.Header{},
					URL:    &url.URL{Path: "/update/123"},
					Body:   formReader(url.Values{"version": {"2"}}),
				}
				updateErr := ErrInvalidChannel
				gomock.InOrder(
					mckStore.EXPECT().KeyToIDs("123").Return("123", "456", nil),
					mckStat.EXPECT().Increment("updates.appserver.incoming"),
					mckStore.EXPECT().Update("123", "456", int64(2)).Return(updateErr),
					mckStat.EXPECT().Increment("updates.appserver.error"),
				)
				eh.ServeMux().ServeHTTP(resp, req)

				So(resp.Code, ShouldEqual, updateErr.Status())
				body, isJSON := getJSON(resp.HeaderMap, resp.Body)
				So(isJSON, ShouldBeTrue)
				So(body.String(), ShouldEqual, `"Could not update channel version"`)
			})
		})

		Convey("Should always route updates if `AlwaysRoute` is enabled", func() {
			eh := NewEndpointHandler()
			eh.setApp(app)
			eh.alwaysRoute = true
			app.SetEndpointHandler(eh)

			uaid := "6952a68ee0e7444ebc54f935c4444b13"
			app.AddWorker(uaid, mckWorker)

			chid := "b7ede546585f4cc9b95e9340e3406951"
			version := int64(1)
			data := "Happy, happy, joy, joy!"

			Convey("And router delivery fails, local succeeds", func() {
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, version,
						gomock.Any(), "", data).Return(false, nil),
					mckStat.EXPECT().Increment("router.broadcast.miss"),
					mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
					mckWorker.EXPECT().Send(chid, version, data).Return(nil),
					mckStat.EXPECT().Increment("updates.appserver.received"),
				)

				ok := eh.deliver(nil, uaid, chid, version, "", data)
				So(ok, ShouldBeTrue)
			})

			Convey("And router delivery succeeds, local succeeds", func() {
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, version,
						gomock.Any(), "", data).Return(true, nil),
					mckStat.EXPECT().Increment("router.broadcast.hit"),
					mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
					mckWorker.EXPECT().Send(chid, version, data).Return(nil),
					mckStat.EXPECT().Increment("updates.appserver.received"),
				)

				ok := eh.deliver(nil, uaid, chid, version, "", data)
				So(ok, ShouldBeTrue)
			})

			Convey("And router delivery succeeds, local fails", func() {
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, version,
						gomock.Any(), "", data).Return(true, nil),
					mckStat.EXPECT().Increment("router.broadcast.hit"),
					mckStat.EXPECT().Timer("updates.routed.hits", gomock.Any()),
					mckWorker.EXPECT().Send(chid, version, data).Return(
						errors.New("client gone")),
					mckStat.EXPECT().Increment("updates.appserver.received"),
				)

				ok := eh.deliver(nil, uaid, chid, version, "", data)
				So(ok, ShouldBeTrue)
			})

			Convey("And router/local delivery fails", func() {
				gomock.InOrder(
					mckStat.EXPECT().Increment("updates.routed.outgoing"),
					mckRouter.EXPECT().Route(nil, uaid, chid, version,
						gomock.Any(), "", data).Return(false, nil),
					mckStat.EXPECT().Increment("router.broadcast.miss"),
					mckStat.EXPECT().Timer("updates.routed.misses", gomock.Any()),
					mckWorker.EXPECT().Send(chid, version, data).Return(
						errors.New("client gone")),
					mckStat.EXPECT().Increment("updates.appserver.rejected"),
				)

				ok := eh.deliver(nil, uaid, chid, version, "", data)
				So(ok, ShouldBeFalse)
			})

		})
	})
}
Example #23
0
func TestEndpointPinger(t *testing.T) {
	useMockFuncs()
	defer useStdFuncs()

	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()

	mckStat := NewMockStatistician(mockCtrl)
	mckPinger := NewMockPropPinger(mockCtrl)
	mckStore := NewMockStore(mockCtrl)
	mckWorker := NewMockWorker(mockCtrl)

	Convey("Proprietary pings", t, func() {
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)
		app.SetPropPinger(mckPinger)
		app.SetStore(mckStore)

		eh := NewEndpointHandler()
		eh.setApp(app)
		eh.setMaxDataLen(4096)
		app.SetEndpointHandler(eh)

		Convey("Should return early if the pinger can bypass the WebSocket", func() {
			uaid := "91357e1a34714cadacb3f13cf47a2736"
			app.AddWorker(uaid, mckWorker)

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   nil,
			}
			gomock.InOrder(
				mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
				mckStat.EXPECT().Increment("updates.appserver.incoming"),
				mckPinger.EXPECT().Send(uaid, int64(1257894000), "").Return(true, nil),
				mckPinger.EXPECT().CanBypassWebsocket().Return(true),
				mckStat.EXPECT().Increment("updates.appserver.received"),
				mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
			)
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 200)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, "{}")
		})

		Convey("Should continue if the pinger cannot bypass the WebSocket", func() {
			uaid := "e3fc2cf1dc44424685010148b076d08b"
			app.AddWorker(uaid, mckWorker)

			data := randomText(eh.maxDataLen)
			vals := make(url.Values)
			vals.Set("data", data)

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}
			gomock.InOrder(
				mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
				mckStat.EXPECT().Increment("updates.appserver.incoming"),
				mckPinger.EXPECT().Send(uaid, int64(1257894000), data).Return(true, nil),
				mckPinger.EXPECT().CanBypassWebsocket().Return(false),
				mckStore.EXPECT().Update(uaid, "456", int64(1257894000)),
				mckWorker.EXPECT().Send("456", int64(1257894000), data),
				mckStat.EXPECT().Increment("updates.appserver.received"),
				mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
			)
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 200)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, "{}")
		})

		Convey("Should continue if the pinger fails", func() {
			uaid := "8f412f5cb2384183bf60f7da26737271"
			app.AddWorker(uaid, mckWorker)

			vals := make(url.Values)
			vals.Set("version", "7")
			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}
			gomock.InOrder(
				mckStore.EXPECT().KeyToIDs("123").Return(uaid, "456", nil),
				mckStat.EXPECT().Increment("updates.appserver.incoming"),
				mckPinger.EXPECT().Send(uaid, int64(7), "").Return(
					true, errors.New("oops")),
				mckStore.EXPECT().Update(uaid, "456", int64(7)),
				mckWorker.EXPECT().Send("456", int64(7), ""),
				mckStat.EXPECT().Increment("updates.appserver.received"),
				mckStat.EXPECT().Timer("updates.handled", gomock.Any()),
			)
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 200)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, "{}")
		})
	})
}
Example #24
0
func TestEndpointInvalidParams(t *testing.T) {
	mockCtrl := gomock.NewController(t)
	defer mockCtrl.Finish()

	mckLogger := NewMockLogger(mockCtrl)
	mckLogger.EXPECT().ShouldLog(gomock.Any()).Return(true).AnyTimes()
	mckLogger.EXPECT().Log(gomock.Any(), gomock.Any(),
		gomock.Any(), gomock.Any()).AnyTimes()

	mckStat := NewMockStatistician(mockCtrl)

	Convey("Invalid update parameters", t, func() {
		app := NewApplication()
		app.SetLogger(mckLogger)
		app.SetMetrics(mckStat)

		eh := NewEndpointHandler()
		eh.setApp(app)
		eh.setMaxDataLen(512)
		app.SetEndpointHandler(eh)

		Convey("Should require PUT requests", func() {
			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "POST",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
			}
			mckStat.EXPECT().Increment("updates.appserver.invalid")
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 405)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Method Not Allowed"`)
		})

		Convey("Should reject negative versions", func() {
			vals := make(url.Values)
			vals.Set("version", "-1")

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}
			mckStat.EXPECT().Increment("updates.appserver.invalid")
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 400)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Invalid Version"`)
		})

		Convey("Should reject invalid versions", func() {
			vals := make(url.Values)
			vals.Set("version", "abc")

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}
			mckStat.EXPECT().Increment("updates.appserver.invalid")
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 400)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Invalid Version"`)
		})

		Convey("Should reject oversized payloads", func() {
			vals := make(url.Values)
			vals.Set("data", randomText(eh.maxDataLen+1))

			resp := httptest.NewRecorder()
			req := &http.Request{
				Method: "PUT",
				Header: http.Header{},
				URL:    &url.URL{Path: "/update/123"},
				Body:   formReader(vals),
			}

			mckStat.EXPECT().Increment("updates.appserver.toolong")
			eh.ServeMux().ServeHTTP(resp, req)

			So(resp.Code, ShouldEqual, 413)
			body, isJSON := getJSON(resp.HeaderMap, resp.Body)
			So(isJSON, ShouldBeTrue)
			So(body.String(), ShouldEqual, `"Data exceeds max length of 512 bytes"`)
		})
	})
}
Example #25
0
func FilePollingInputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)

	tmpFileName := fmt.Sprintf("filepollinginput-test-%d", time.Now().UnixNano())
	tmpFilePath := filepath.Join(os.TempDir(), tmpFileName)

	defer func() {
		ctrl.Finish()
		os.Remove(tmpFilePath)
	}()

	pConfig := NewPipelineConfig(nil)
	var wg sync.WaitGroup
	errChan := make(chan error, 1)
	bytesChan := make(chan []byte, 1)
	tickChan := make(chan time.Time)

	retPackChan := make(chan *PipelinePack, 2)
	defer close(retPackChan)

	c.Specify("A FilePollingInput", func() {
		input := new(FilePollingInput)

		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
		ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)

		config := input.ConfigStruct().(*FilePollingInputConfig)
		config.FilePath = tmpFilePath

		startInput := func(msgCount int) {
			wg.Add(1)
			go func() {
				errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
				wg.Done()
			}()
		}

		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)
		ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

		c.Specify("gets updated information when reading a file", func() {

			err := input.Init(config)
			c.Assume(err, gs.IsNil)

			ith.MockInputRunner.EXPECT().NewSplitterRunner("").Return(ith.MockSplitterRunner)
			ith.MockSplitterRunner.EXPECT().UseMsgBytes().Return(false)
			ith.MockSplitterRunner.EXPECT().SetPackDecorator(gomock.Any())
			splitCall := ith.MockSplitterRunner.EXPECT().SplitStream(gomock.Any(),
				nil).Return(io.EOF).Times(2)
			splitCall.Do(func(f *os.File, del Deliverer) {
				fBytes, err := ioutil.ReadAll(f)
				if err != nil {
					fBytes = []byte(err.Error())
				}
				bytesChan <- fBytes
			})

			startInput(2)

			f, err := os.Create(tmpFilePath)
			c.Expect(err, gs.IsNil)

			_, err = f.Write([]byte("test1"))
			c.Expect(err, gs.IsNil)
			c.Expect(f.Close(), gs.IsNil)

			tickChan <- time.Now()

			msgBytes := <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "test1")

			f, err = os.Create(tmpFilePath)
			c.Expect(err, gs.IsNil)

			_, err = f.Write([]byte("test2"))
			c.Expect(err, gs.IsNil)
			c.Expect(f.Close(), gs.IsNil)

			tickChan <- time.Now()
			msgBytes = <-bytesChan
			c.Expect(string(msgBytes), gs.Equals, "test2")

			input.Stop()
			wg.Wait()
			c.Expect(<-errChan, gs.IsNil)
		})

	})

}
Example #26
0
func FilterSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	fth := NewFilterTestHelper(ctrl)
	inChan := make(chan *pipeline.PipelinePack, 1)
	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxFilter", func() {
		sbFilter := new(SandboxFilter)
		sbFilter.SetPipelineConfig(pConfig)
		config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
		config.MemoryLimit = 32000
		config.InstructionLimit = 1000
		config.OutputLimit = 1024

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("Uninitialized", func() {
			err := sbFilter.ReportMsg(msg)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Over inject messages from ProcessMessage", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(2)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(2)

			config.ScriptFilename = "../lua/testsupport/processinject.lua"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			termErr := pipeline.TerminatedError("exceeded InjectMessage count")
			c.Expect(err.Error(), gs.Equals, termErr.Error())
		})

		c.Specify("Over inject messages from TimerEvent", func() {
			var timer <-chan time.Time
			timer = time.Tick(time.Duration(1) * time.Millisecond)
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(11)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(11)

			config.ScriptFilename = "../lua/testsupport/timerinject.lua"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			go func() {
				time.Sleep(time.Duration(250) * time.Millisecond)
				close(inChan)
			}()
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			termErr := pipeline.TerminatedError("exceeded InjectMessage count")
			c.Expect(err.Error(), gs.Equals, termErr.Error())
		})

		c.Specify("Preserves data", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)

			config.ScriptFilename = "../lua/testsupport/serialize.lua"
			config.PreserveData = true
			sbFilter.SetName("serialize")
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			_, err = os.Stat("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
			err = os.Remove("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
		})
	})

	c.Specify("A SandboxManagerFilter", func() {
		pConfig.Globals.BaseDir = os.TempDir()
		sbxMgrsDir := filepath.Join(pConfig.Globals.BaseDir, "sbxmgrs")
		defer func() {
			tmpErr := os.RemoveAll(sbxMgrsDir)
			c.Expect(tmpErr, gs.IsNil)
		}()

		sbmFilter := new(SandboxManagerFilter)
		sbmFilter.SetPipelineConfig(pConfig)
		config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
		config.MaxFilters = 1

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("Control message in the past", func() {
			sbmFilter.Init(config)
			pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: 5 seconds skew"))
			inChan <- pack
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Control message in the future", func() {
			sbmFilter.Init(config)
			pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: -5 seconds skew"))
			inChan <- pack
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Generates the right default working directory", func() {
			sbmFilter.Init(config)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			name := "SandboxManagerFilter"
			fth.MockFilterRunner.EXPECT().Name().Return(name)
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
			_, err = os.Stat(sbxMgrsDir)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Sanity check the default sandbox configuration limits", func() {
			sbmFilter.Init(config)
			c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
			c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
			c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
		})

		c.Specify("Sanity check the user specified sandbox configuration limits", func() {
			config.MemoryLimit = 123456
			config.InstructionLimit = 4321
			config.OutputLimit = 8765
			sbmFilter.Init(config)
			c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
			c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
			c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
		})

		c.Specify("Creates a SandboxFilter runner", func() {
			sbxName := "SandboxFilter"
			sbxMgrName := "SandboxManagerFilter"
			code := `
			require("cjson")

			function process_message()
			    inject_payload(cjson.encode({a = "b"}))
			    return 0
			end
			`
			cfg := `
			[%s]
			type = "SandboxFilter"
			message_matcher = "TRUE"
			script_type = "lua"
			`
			cfg = fmt.Sprintf(cfg, sbxName)
			msg.SetPayload(code)
			f, err := message.NewField("config", cfg, "toml")
			c.Assume(err, gs.IsNil)
			msg.AddField(f)

			fMatchChan := pConfig.Router().AddFilterMatcher()
			errChan := make(chan error)

			fth.MockFilterRunner.EXPECT().Name().Return(sbxMgrName)
			fullSbxName := fmt.Sprintf("%s-%s", sbxMgrName, sbxName)
			fth.MockHelper.EXPECT().Filter(fullSbxName).Return(nil, false)
			fth.MockFilterRunner.EXPECT().LogMessage(fmt.Sprintf("Loading: %s", fullSbxName))

			sbmFilter.Init(config)
			go func() {
				err := sbmFilter.loadSandbox(fth.MockFilterRunner, fth.MockHelper, sbxMgrsDir,
					msg)
				errChan <- err
			}()

			fMatch := <-fMatchChan
			c.Expect(fMatch.MatcherSpecification().String(), gs.Equals, "TRUE")
			c.Expect(<-errChan, gs.IsNil)

			go func() {
				<-pConfig.Router().RemoveFilterMatcher()
			}()
			ok := pConfig.RemoveFilterRunner(fullSbxName)
			c.Expect(ok, gs.IsTrue)
		})
	})

	c.Specify("A Cpu Stats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "cpustats"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/cpustats.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		msg := getTestMessage()
		fields := make([]*message.Field, 4)
		fields[0], _ = message.NewField("1MinAvg", 0.08, "")
		fields[1], _ = message.NewField("5MinAvg", 0.04, "")
		fields[2], _ = message.NewField("15MinAvg", 0.02, "")
		fields[3], _ = message.NewField("NumProcesses", 5, "")
		msg.Fields = fields

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().Name().Return("cpustats")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with cpuload data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			for i := 1; i <= 3; i++ {
				// Fill in the data
				t := int64(i * 1000000000)
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":1,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"1MinAvg","unit":"Count","aggregation":"max"},{"name":"5MinAvg","unit":"Count","aggregation":"max"},{"name":"15MinAvg","unit":"Count","aggregation":"max"},{"name":"NumProcesses","unit":"Count","aggregation":"max"}]}
0.08	0.04	0.02	5
0.08	0.04	0.02	5
0.08	0.04	0.02	5
`

			c.Expect(p.Message.GetPayload(), gs.Equals, pl)
		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("A Memstats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "memstats"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/memstats.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		msg := getTestMessage()
		field_names := []string{"MemFree", "Cached", "Active", "Inactive", "VmallocUsed", "Shmem", "SwapCached", "SwapTotal", "SwapFree"}
		fields := make([]*message.Field, len(field_names))
		for i, name := range field_names {
			fields[i], _ = message.NewField(name, 100, "")
		}
		msg.Fields = fields

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().Name().Return("memstats")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with memstats data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			for i := 1; i <= 3; i++ {
				// Fill in the data
				t := int64(i * 1000000000)
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":1,"rows":3,"columns":9,"seconds_per_row":1,"column_info":[{"name":"MemFree","unit":"Count","aggregation":"max"},{"name":"Cached","unit":"Count","aggregation":"max"},{"name":"Active","unit":"Count","aggregation":"max"},{"name":"Inactive","unit":"Count","aggregation":"max"},{"name":"VmallocUsed","unit":"Count","aggregation":"max"},{"name":"Shmem","unit":"Count","aggregation":"max"},{"name":"SwapCached","unit":"Count","aggregation":"max"},{"name":"SwapFree","unit":"Count","aggregation":"max"},{"name":"SwapUsed","unit":"Count","aggregation":"max"}]}
100	100	100	100	100	100	100	100	0
100	100	100	100	100	100	100	100	0
100	100	100	100	100	100	100	100	0
`
			c.Expect(p.Message.GetPayload(), gs.Equals, pl)

		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("A diskstats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "diskstats"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/diskstats.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retMsgChan := make(chan *message.Message, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retMsgChan)
		}()

		msg := getTestMessage()

		field_names := []string{
			"WritesCompleted",
			"ReadsCompleted",
			"SectorsWritten",
			"SectorsRead",
			"WritesMerged",
			"ReadsMerged",
			"TimeWriting",
			"TimeReading",
			"TimeDoingIO",
			"WeightedTimeDoingIO",
		}

		num_fields := len(field_names) + 1
		fields := make([]*message.Field, num_fields)
		msg.Fields = fields
		timeInterval, _ := message.NewField("TickerInterval", 1, "")
		fields[num_fields-1] = timeInterval
		fieldVal := 100

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelineConfig().AnyTimes()
		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).AnyTimes()
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer).AnyTimes()
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan).AnyTimes()
		fth.MockFilterRunner.EXPECT().Name().Return("diskstats").AnyTimes()
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			msg := pack.Message
			pack.Message = new(message.Message)
			retMsgChan <- msg
		}).Return(true).AnyTimes()

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with diskstats data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			// Iterate 4 times since the first one doesn't actually set the cbuf
			// in order to set the delta in the cbuf
			for i := 1; i <= 4; i++ {
				// Fill in the fields
				for i, name := range field_names {
					fields[i], _ = message.NewField(name, fieldVal, "")
				}
				// Scale up the value so we can see the delta growing
				// by 100 each iteration
				fieldVal += i * 100

				t := int64(i * 1000000000)
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			testExpects := map[string]string{
				"Time doing IO": `{"time":2,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"TimeWriting","unit":"ms","aggregation":"max"},{"name":"TimeReading","unit":"ms","aggregation":"max"},{"name":"TimeDoingIO","unit":"ms","aggregation":"max"},{"name":"WeightedTimeDoi","unit":"ms","aggregation":"max"}]}
200	200	200	200
400	400	400	400
700	700	700	700
`,
				"Disk Stats": `{"time":2,"rows":3,"columns":6,"seconds_per_row":1,"column_info":[{"name":"WritesCompleted","unit":"per_1_s","aggregation":"none"},{"name":"ReadsCompleted","unit":"per_1_s","aggregation":"none"},{"name":"SectorsWritten","unit":"per_1_s","aggregation":"none"},{"name":"SectorsRead","unit":"per_1_s","aggregation":"none"},{"name":"WritesMerged","unit":"per_1_s","aggregation":"none"},{"name":"ReadsMerged","unit":"per_1_s","aggregation":"none"}]}
100	100	100	100	100	100
200	200	200	200	200	200
300	300	300	300	300	300
`,
			}

			timer <- time.Now()
			for i := 0; i < 2; i++ {
				m := <-retMsgChan
				name, ok := m.GetFieldValue("payload_name")
				c.Assume(ok, gs.IsTrue)
				nameVal, ok := name.(string)
				c.Assume(ok, gs.IsTrue)
				c.Expect(m.GetPayload(), gs.Equals, testExpects[nameVal])
			}
		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("http_status filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "http_status"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/http_status.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(2)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		field, _ := message.NewField("status", 0, "")
		msg := &message.Message{}
		msg.SetTimestamp(0)
		msg.AddField(field)

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().Name().Return("http_status")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with http status data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			for i := 0; i <= 6; i++ {
				msg.Fields[0].ValueInteger[0] = int64(i * 100) // iterate through the status codes with a bogus status on each end
				pack.Message = msg
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":0,"rows":2,"columns":6,"seconds_per_row":1,"column_info":[{"name":"HTTP_100","unit":"count","aggregation":"sum"},{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1	1	1	1	1	2
nan	nan	nan	nan	nan	nan
`
			c.Expect(p.Message.GetPayload(), gs.Equals, pl)

		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})
}
Example #27
0
func DecoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxDecoder", func() {

		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)

		c.Specify("that uses lpeg and inject_message", func() {
			dRunner.EXPECT().Name().Return("serialize")
			conf.ScriptFilename = "../lua/testsupport/decoder.lua"
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			c.Specify("decodes simple messages", func() {
				data := "1376389920 debug id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				_, err = decoder.Decode(pack)
				c.Assume(err, gs.IsNil)

				c.Expect(pack.Message.GetTimestamp(),
					gs.Equals,
					int64(1376389920000000000))

				c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

				var ok bool
				var value interface{}
				value, ok = pack.Message.GetFieldValue("id")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "2321")

				value, ok = pack.Message.GetFieldValue("url")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "example.com")

				value, ok = pack.Message.GetFieldValue("item")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "1")
				decoder.Shutdown()
			})

			c.Specify("decodes an invalid messages", func() {
				data := "1376389920 bogus id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(len(packs), gs.Equals, 0)
				c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
				c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
				decoder.Shutdown()
			})

			c.Specify("Preserves data", func() {
				conf.ScriptFilename = "../lua/testsupport/serialize.lua"
				conf.PreserveData = true
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)
				decoder.Shutdown()
				_, err = os.Stat("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
				err = os.Remove("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("that only uses write_message", func() {
			conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
			dRunner.EXPECT().Name().Return("write_message")
			err := decoder.Init(conf)
			decoder.SetDecoderRunner(dRunner)
			c.Assume(err, gs.IsNil)

			c.Specify("adds a string field to the message", func() {
				data := "string field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(string), gs.Equals, "foo")
			})

			c.Specify("adds a numeric field to the message", func() {
				data := "num field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(float64), gs.Equals, float64(1))
			})

			c.Specify("adds a boolean field to the message", func() {
				data := "bool field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(bool), gs.Equals, true)
			})

			c.Specify("sets type and payload", func() {
				data := "set type and payload"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
				c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
			})

			c.Specify("sets field value with representation", func() {
				data := "set field value with representation"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("rep")
				c.Expect(len(fields), gs.Equals, 1)
				field := fields[0]
				values := field.GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "foo")
				c.Expect(field.GetRepresentation(), gs.Equals, "representation")
			})

			c.Specify("sets multiple field string values", func() {
				data := "set multiple field string values"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("multi")
				c.Expect(len(fields), gs.Equals, 2)
				values := fields[0].GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "first")
				values = fields[1].GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "second")
			})

			c.Specify("sets field string array value", func() {
				data := "set field string array value"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("array")
				c.Expect(len(fields), gs.Equals, 1)
				values := fields[0].GetValueString()
				c.Expect(len(values), gs.Equals, 2)
				c.Expect(values[0], gs.Equals, "first")
				c.Expect(values[1], gs.Equals, "second")
			})
		})
	})

	c.Specify("A Multipack SandboxDecoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
		supply := make(chan *pipeline.PipelinePack, 3)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message = getTestMessage()

		pack1 := pipeline.NewPipelinePack(supply)
		pack2 := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")

		c.Specify("decodes into multiple packs", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			decoder.SetDecoderRunner(dRunner)
			gomock.InOrder(
				dRunner.EXPECT().NewPack().Return(pack1),
				dRunner.EXPECT().NewPack().Return(pack2),
			)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 3)
			c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
			c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
			c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")

			for i := 0; i < 1; i++ {
				c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
				c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
				c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
				c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))

			}
			decoder.Shutdown()
		})
	})

	c.Specify("Linux Cpu Stats decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/linux_loadavg.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes a message", func() {
			payload := "0.00 0.01 0.05 3/153 660\n"
			pack.Message.SetPayload(payload)
			f, err := message.NewField("FilePath", "/proc/loadavg", "")
			c.Assume(err, gs.IsNil)
			pack.Message.AddField(f)

			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)
			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("1MinAvg")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 0.00)

			value, ok = pack.Message.GetFieldValue("5MinAvg")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 0.01)

			value, ok = pack.Message.GetFieldValue("15MinAvg")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 0.05)

			value, ok = pack.Message.GetFieldValue("NumProcesses")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(3))

			value, ok = pack.Message.GetFieldValue("FilePath")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, "/proc/loadavg")
		})

		c.Specify("decodes an invalid message", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Linux Mem Stats decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/linux_memstats.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes a message", func() {
			payload := `MemTotal:        4047616 kB
MemFree:         3135780 kB
HugePages_Free:        0
`
			pack.Message.SetPayload(payload)
			f, err := message.NewField("FilePath", "/proc/meminfo", "")
			c.Assume(err, gs.IsNil)
			pack.Message.AddField(f)

			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)
			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("MemTotal")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 4.047616e+06)

			value, ok = pack.Message.GetFieldValue("MemFree")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 3.13578e+06)

			value, ok = pack.Message.GetFieldValue("HugePages_Free")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("FilePath")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, "/proc/meminfo")
		})

		c.Specify("decodes an invalid message", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Linux Disk Stats decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/linux_diskstats.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes a message", func() {
			payload := "   13903    11393   969224    49444    10780    10161  1511920     4104        0     5064    53468\n"
			pack.Message.SetPayload(payload)
			f, err := message.NewField("FilePath", "/sys/block/sda/stat", "")
			c.Assume(err, gs.IsNil)
			pack.Message.AddField(f)

			f, err = message.NewField("TickerInterval", int64(2), "")
			c.Assume(err, gs.IsNil)
			pack.Message.AddField(f)

			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)
			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			// These are in the same order the payload should be
			value, ok = pack.Message.GetFieldValue("ReadsCompleted")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(13903))

			value, ok = pack.Message.GetFieldValue("ReadsMerged")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(11393))

			value, ok = pack.Message.GetFieldValue("SectorsRead")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(969224))

			value, ok = pack.Message.GetFieldValue("TimeReading")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(49444))

			value, ok = pack.Message.GetFieldValue("WritesCompleted")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(10780))

			value, ok = pack.Message.GetFieldValue("WritesMerged")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(10161))

			value, ok = pack.Message.GetFieldValue("SectorsWritten")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(1511920))

			value, ok = pack.Message.GetFieldValue("TimeWriting")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(4104))

			value, ok = pack.Message.GetFieldValue("NumIOInProgress")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("TimeDoingIO")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(5064))

			value, ok = pack.Message.GetFieldValue("WeightedTimeDoingIO")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(53468))

			value, ok = pack.Message.GetFieldValue("TickerInterval")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(2))

			value, ok = pack.Message.GetFieldValue("FilePath")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, "/sys/block/sda/stat")
		})

		c.Specify("decodes a message with no leading space", func() {
			payload := "19092852        0 510563170 15817012 46452019        0 1546950712 262535124        0 23823976 278362684\n"
			pack.Message.SetPayload(payload)

			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			value, ok := pack.Message.GetFieldValue("ReadsCompleted")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(19092852))
		})

		c.Specify("decodes an invalid message", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Nginx access log decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
		conf.Config["user_agent_transform"] = true
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392050801000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("remote_addr")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "127.0.0.1")

			value, ok = pack.Message.GetFieldValue("user_agent_browser")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Firefox")
			value, ok = pack.Message.GetFieldValue("user_agent_version")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(26))
			value, ok = pack.Message.GetFieldValue("user_agent_os")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Linux")
			_, ok = pack.Message.GetFieldValue("http_user_agent")
			c.Expect(ok, gs.Equals, false)

			value, ok = pack.Message.GetFieldValue("body_bytes_sent")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("status")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(304))
			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Apache access log decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/apache_access.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
		conf.Config["user_agent_transform"] = true
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392050801000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("remote_addr")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "127.0.0.1")

			value, ok = pack.Message.GetFieldValue("user_agent_browser")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Firefox")
			value, ok = pack.Message.GetFieldValue("user_agent_version")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(26))
			value, ok = pack.Message.GetFieldValue("user_agent_os")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Linux")
			_, ok = pack.Message.GetFieldValue("http_user_agent")
			c.Expect(ok, gs.Equals, false)

			value, ok = pack.Message.GetFieldValue("body_bytes_sent")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("status")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(304))
			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("rsyslog decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["type"] = "MyTestFormat"
		conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
		conf.Config["tz"] = "America/Los_Angeles"
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n"
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			// Syslog timestamp doesn't support year, so we have to calculate
			// it for the current year or else this test will fail every
			// January.
			year := time.Now().Year()
			tStr := fmt.Sprintf("%d Feb 10 12:58:58 -0800", year)
			t, err := time.Parse("2006 Jan 02 15:04:05 -0700", tStr)
			c.Assume(err, gs.IsNil)
			unixT := t.UnixNano()

			c.Expect(pack.Message.GetTimestamp(), gs.Equals, unixT)

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
			c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
			c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.")
			c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"])

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("programname")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "widget")

			value, ok = pack.Message.GetFieldValue("syslogfacility")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(3))

			value, ok = pack.Message.GetFieldValue("timegenerated")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(1392065939000000000))

			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("mysql decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["truncate_sql"] = int64(5)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decode standard slow query log", func() {
			data := `# User@Host: syncrw[syncrw] @  [127.0.0.1]
# Query_time: 2.964652  Lock_time: 0.000050 Rows_sent: 251  Rows_examined: 9773
use widget;
SET last_insert_id=999,insert_id=1000,timestamp=1399500744;
# administrator command: do something
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;`
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1399500744000000000))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
			c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query")

			decoder.Shutdown()
		})
	})

	c.Specify("mariadb decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["truncate_sql"] = int64(5)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decode standard slow query log", func() {
			data := `# User@Host: syncrw[syncrw] @  [127.0.0.1]
# Thread_id: 110804  Schema: weave0  QC_hit: No
# Query_time: 1.178108  Lock_time: 0.000053  Rows_sent: 198  Rows_examined: 198
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;`
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1399500744000000000))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
			c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query")

			decoder.Shutdown()
		})
	})
}
func ProcessDirectoryInputSpec(c gs.Context) {

	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil)
	ith := new(plugins_ts.InputTestHelper)
	ith.Msg = pipeline_ts.GetTestMessage()
	ith.Pack = NewPipelinePack(pConfig.InputRecycleChan())

	// set up mock helper, decoder set, and packSupply channel
	ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)
	ith.MockDeliverer = pipelinemock.NewMockDeliverer(ctrl)
	ith.MockSplitterRunner = pipelinemock.NewMockSplitterRunner(ctrl)
	ith.PackSupply = make(chan *PipelinePack, 1)

	ith.PackSupply <- ith.Pack

	err := pConfig.RegisterDefault("NullSplitter")
	c.Assume(err, gs.IsNil)

	c.Specify("A ProcessDirectoryInput", func() {
		pdiInput := ProcessDirectoryInput{}
		pdiInput.SetPipelineConfig(pConfig)

		config := pdiInput.ConfigStruct().(*ProcessDirectoryInputConfig)
		workingDir, err := os.Getwd()
		c.Assume(err, gs.IsNil)
		config.ProcessDir = filepath.Join(workingDir, "testsupport", "processes")

		// `Ticker` is the last thing called during the setup part of the
		// input's `Run` method, so it triggers a waitgroup that tests can
		// wait on when they need to ensure initialization has finished.
		var started sync.WaitGroup
		started.Add(1)
		tickChan := make(chan time.Time, 1)
		ith.MockInputRunner.EXPECT().Ticker().Return(tickChan).Do(
			func() {
				started.Done()
			})

		// Similarly we use a waitgroup to signal when LogMessage has been
		// called to know when reloads have completed. Warning: If you call
		// expectLogMessage with a msg that is never passed to LogMessage and
		// then you call loaded.Wait() then your test will hang and never
		// complete.
		var loaded sync.WaitGroup
		expectLogMessage := func(msg string) {
			loaded.Add(1)
			ith.MockInputRunner.EXPECT().LogMessage(msg).Do(
				func(msg string) {
					loaded.Done()
				})
		}

		// Same name => same content.
		paths := []string{
			filepath.Join(config.ProcessDir, "100", "h0.toml"),
			filepath.Join(config.ProcessDir, "100", "h1.toml"),
			filepath.Join(config.ProcessDir, "200", "h0.toml"),
			filepath.Join(config.ProcessDir, "300", "h1.toml"),
		}

		copyFile := func(src, dest string) {
			inFile, err := os.Open(src)
			c.Assume(err, gs.IsNil)
			outFile, err := os.Create(dest)
			c.Assume(err, gs.IsNil)
			_, err = io.Copy(outFile, inFile)
			c.Assume(err, gs.IsNil)
			inFile.Close()
			outFile.Close()
		}

		err = pdiInput.Init(config)
		c.Expect(err, gs.IsNil)

		for _, p := range paths {
			expectLogMessage("Added: " + p)
		}
		go pdiInput.Run(ith.MockInputRunner, ith.MockHelper)
		defer func() {
			pdiInput.Stop()
			for _, entry := range pdiInput.inputs {
				entry.ir.Input().Stop()
			}
		}()
		started.Wait()

		c.Specify("loads scheduled jobs", func() {
			pathIndex := func(name string) (i int) {
				var p string
				for i, p = range paths {
					if name == p {
						return
					}
				}
				return -1
			}

			for name, entry := range pdiInput.inputs {
				i := pathIndex(name)
				// Make sure each file path got registered.
				c.Expect(i, gs.Not(gs.Equals), -1)
				dirName := filepath.Base(filepath.Dir(name))
				dirInt, err := strconv.Atoi(dirName)
				c.Expect(err, gs.IsNil)
				// And that the ticker interval was read correctly.
				c.Expect(uint(dirInt), gs.Equals, entry.config.TickerInterval)
			}
		})

		c.Specify("discovers and adds a new job", func() {
			// Copy one of the files to register a new process.
			newPath := filepath.Join(config.ProcessDir, "300", "h0.toml")
			copyFile(paths[0], newPath)
			defer func() {
				err := os.Remove(newPath)
				c.Assume(err, gs.IsNil)
			}()

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Added: " + newPath)
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure our plugin was loaded.
			c.Expect(len(pdiInput.inputs), gs.Equals, 5)
			newEntry, ok := pdiInput.inputs[newPath]
			c.Expect(ok, gs.IsTrue)
			c.Expect(newEntry.config.TickerInterval, gs.Equals, uint(300))
		})

		c.Specify("removes a deleted job", func() {
			err := os.Remove(paths[3])
			c.Assume(err, gs.IsNil)
			defer func() {
				copyFile(paths[1], paths[3])
			}()

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Removed: " + paths[3])
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure our plugin was deleted.
			c.Expect(len(pdiInput.inputs), gs.Equals, 3)
		})

		c.Specify("notices a changed job", func() {
			// Overwrite one job w/ a slightly different one.
			copyFile(paths[0], paths[3])
			defer copyFile(paths[1], paths[3])

			// Set up expectations and trigger process dir reload.
			expectLogMessage("Removed: " + paths[3])
			expectLogMessage("Added: " + paths[3])
			tickChan <- time.Now()
			loaded.Wait()

			// Make sure the new config was loaded.
			c.Expect(pdiInput.inputs[paths[3]].config.Command["0"].Args[0], gs.Equals,
				"hello world\n")
		})
	})

}
Example #29
0
func DashboardOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := pipeline.NewPipelineConfig(nil)

	dashboardOutput := new(DashboardOutput)
	dashboardOutput.pConfig = pConfig

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)

	errChan := make(chan error, 1)

	startOutput := func() {
		go func() {
			errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
		}()
	}

	if runtime.GOOS != "windows" {
		c.Specify("A DashboardOutput", func() {

			tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
			c.Assume(err, gs.IsNil)
			config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
			config.WorkingDirectory = tmpdir

			c.Specify("Init halts if basedirectory is not writable", func() {
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)
				err = dashboardOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
			})

			c.Specify("that is running", func() {
				startedChan := make(chan bool, 1)
				defer close(startedChan)
				ts := httptest.NewUnstartedServer(nil)

				dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
					ts.Start()
					startedChan <- true
					return nil
				}

				ticker := make(chan time.Time)
				inChan := make(chan *pipeline.PipelinePack, 1)
				recycleChan := make(chan *pipeline.PipelinePack, 1)
				pack := pipeline.NewPipelinePack(recycleChan)
				pack.Message = pipeline_ts.GetTestMessage()

				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)

				err := os.MkdirAll(tmpdir, 0700)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)

				dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					// Noop
				})

				c.Specify("sets custom http headers", func() {
					config.Headers = http.Header{
						"One":  []string{"two", "three"},
						"Four": []string{"five", "six", "seven"},
					}
					err = dashboardOutput.Init(config)
					c.Assume(err, gs.IsNil)
					ts.Config = dashboardOutput.server

					startOutput()

					inChan <- pack
					<-startedChan
					resp, err := http.Get(ts.URL)
					c.Assume(err, gs.IsNil)
					resp.Body.Close()
					c.Assume(resp.StatusCode, gs.Equals, 200)

					// Verify headers are there
					eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
					c.Expect(eq, gs.IsTrue)
					eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
					c.Expect(eq, gs.IsTrue)
				})

				close(inChan)
				c.Expect(<-errChan, gs.IsNil)

				ts.Close()
			})
		})
	}
}
Example #30
0
func SmtpOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	var wg sync.WaitGroup
	inChan := make(chan *PipelinePack, 1)
	pConfig := NewPipelineConfig(nil)

	encoder := new(plugins.PayloadEncoder)
	econfig := encoder.ConfigStruct().(*plugins.PayloadEncoderConfig)
	econfig.AppendNewlines = false
	encoder.Init(econfig)

	c.Specify("A SmtpOutput", func() {
		smtpOutput := new(SmtpOutput)

		config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
		config.SendTo = []string{"root"}

		msg := pipeline_ts.GetTestMessage()
		pack := NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
		inChanCall.Return(inChan)
		runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
		runnerName.Return("SmtpOutput")
		oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
		encCall := oth.MockOutputRunner.EXPECT().Encode(pack)

		c.Specify("send email payload message", func() {
			err := smtpOutput.Init(config)
			c.Assume(err, gs.IsNil)
			smtpOutput.sendFunction = testSendMail

			outStr := "Write me out to the network"
			pack.Message.SetPayload(outStr)
			encCall.Return(encoder.Encode(pack))
			wg.Add(1)
			go func() {
				smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
			inChan <- pack
			close(inChan)
			wg.Wait()
		})
	})

	c.Specify("SmtpOutput Message Body Encoding", func() {
		smtpOutput := new(SmtpOutput)
		chars := "123456789012345678901234567890123456789012345678901234567"
		charsE := "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3"
		examples := [][]string{
			{"Hello", "SGVsbG8="},
			{chars, charsE},
			{chars + chars, charsE + "\r\n" + charsE},
			{chars + chars + "Hello", charsE + "\r\n" + charsE + "\r\n" + "SGVsbG8="},
			{"", ""},
			{"1", "MQ=="},
		}
		for _, example := range examples {
			smtpOutput.encodeFullMsg([]byte(example[0]))
			c.Expect(string(smtpOutput.fullMsg), gs.Equals, example[1])
		}
	})

	// // Use this test with a real server
	// c.Specify("Real SmtpOutput output", func() {
	// 	smtpOutput := new(SmtpOutput)

	// 	config := smtpOutput.ConfigStruct().(*SmtpOutputConfig)
	// 	config.SendTo = []string{"root"}

	// 	msg := pipeline_ts.GetTestMessage()
	// 	pack := NewPipelinePack(pConfig.InputRecycleChan())
	// 	pack.Message = msg
	// 	pack.Decoded = true
	// 	inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes()
	// 	inChanCall.Return(inChan)
	// 	runnerName := oth.MockOutputRunner.EXPECT().Name().AnyTimes()
	// 	runnerName.Return("SmtpOutput")
	// 	oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
	// 	encCall := oth.MockOutputRunner.EXPECT().Encode(pack)

	// 	c.Specify("send a real email essage", func() {

	// 		err := smtpOutput.Init(config)
	// 		c.Assume(err, gs.IsNil)

	// 		outStr := "Write me out to the network"
	// 		pack.Message.SetPayload(outStr)
	// 		encCall.Return(encoder.Encode(pack))
	// 		go func() {
	// 			wg.Add(1)
	// 			smtpOutput.Run(oth.MockOutputRunner, oth.MockHelper)
	// 			wg.Done()
	// 		}()
	// 		inChan <- pack
	// 		time.Sleep(1000) // allow time for the message output
	// 		close(inChan)
	// 		wg.Wait()
	// // manually check the mail
	//	})
	// })
}