Esempio n. 1
0
func TestLoadDir(t *testing.T) {
	origGlobals := pipeline.Globals
	origAvailablePlugins := make(map[string]func() interface{})
	for k, v := range pipeline.AvailablePlugins {
		origAvailablePlugins[k] = v
	}

	defer func() {
		pipeline.Globals = origGlobals
		pipeline.AvailablePlugins = origAvailablePlugins
	}()

	pipeConfig := pipeline.NewPipelineConfig(nil)
	confDirPath := "../../plugins/testsupport/config_dir"
	err := loadFullConfig(pipeConfig, &confDirPath)
	if err != nil {
		t.Fatal(err)
	}

	// verify the inputs sections load properly with a custom name
	udp, ok := pipeConfig.InputRunners["UdpInput"]
	if !ok {
		t.Fatal("No UdpInput configured.")
	}
	defer udp.Input().Stop()

	// and the decoders sections load
	_, ok = pipeConfig.DecoderWrappers["ProtobufDecoder"]
	if !ok {
		t.Fatal("No ProtobufDecoder configured.")
	}

	// and the outputs sections load
	_, ok = pipeConfig.OutputRunners["LogOutput"]
	if !ok {
		t.Fatal("No LogOutput configured")
	}

	// and the filters sections load
	_, ok = pipeConfig.FilterRunners["sample"]
	if !ok {
		t.Fatal("No `sample` filter configured.")
	}

	// and the encoders sections load
	encoder, ok := pipeConfig.Encoder("PayloadEncoder", "foo")
	if !ok {
		t.Fatal("No PayloadEncoder configured.")
	}
	_, ok = encoder.(*plugins.PayloadEncoder)
	if !ok {
		t.Fatal("PayloadEncoder isn't a PayloadEncoder")
	}

	// and that the non "*.toml" file did *not* load
	_, ok = pipeConfig.FilterRunners["not_loaded"]
	if ok {
		t.Fatal("`not_loaded` filter *was* loaded, shouldn't have been!")
	}
}
Esempio n. 2
0
func main() {
	configFile := flag.String("config", "agent.conf", "Agent Config file")
	maxprocs := flag.Int("maxprocs", 1, "Go runtime MAXPROCS value")
	poolSize := flag.Int("poolsize", 1000, "Pipeline pool size")
	pprofName := flag.String("pprof", "", "Go profiler output file")
	version := flag.Bool("version", false, "Output version and exit")
	flag.Parse()

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	runtime.GOMAXPROCS(*maxprocs)

	if *pprofName != "" {
		profFile, err := os.Create(*pprofName)
		if err != nil {
			log.Fatalln(err)
		}
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	// Set up and load the pipeline configuration and start the daemon.
	pipeconf := pipeline.NewPipelineConfig(*poolSize)
	err := pipeconf.LoadFromConfigFile(*configFile)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}
Esempio n. 3
0
func main() {
	configFile := flag.String("config", "/etc/hekad.toml", "Config file")
	maxprocs := flag.Int("maxprocs", 1, "Go runtime MAXPROCS value")
	poolSize := flag.Int("poolsize", 100, "Pipeline pool size")
	decoderPoolSize := flag.Int("decoder_poolsize", 4, "Decoder pool size")
	chanSize := flag.Int("plugin_chansize", 50, "Plugin input channel buffer size")
	cpuProfName := flag.String("cpuprof", "", "Go CPU profiler output file")
	memProfName := flag.String("memprof", "", "Go memory profiler output file")
	version := flag.Bool("version", false, "Output version and exit")
	maxMsgLoops := flag.Uint("max_message_loops", 4, "Maximum number of times a message can pass thru the system")
	flag.Parse()

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	runtime.GOMAXPROCS(*maxprocs)

	if *cpuProfName != "" {
		profFile, err := os.Create(*cpuProfName)
		if err != nil {
			log.Fatalln(err)
		}
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if *memProfName != "" {
		defer func() {
			profFile, err := os.Create(*memProfName)
			if err != nil {
				log.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Set up and load the pipeline configuration and start the daemon.
	globals := pipeline.DefaultGlobals()
	globals.PoolSize = *poolSize
	globals.DecoderPoolSize = *decoderPoolSize
	globals.PluginChanSize = *chanSize
	globals.MaxMsgLoops = *maxMsgLoops
	if globals.MaxMsgLoops == 0 {
		globals.MaxMsgLoops = 1
	}
	pipeconf := pipeline.NewPipelineConfig(globals)
	err := pipeconf.LoadFromConfigFile(*configFile)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}
Esempio n. 4
0
func TestCustomHostname(t *testing.T) {
	expected := "my.example.com"
	configPath := "../../pipeline/testsupport/sample-hostname.toml"
	config, err := LoadHekadConfig(configPath)
	if err != nil {
		t.Fatal(err)
	}
	if config.Hostname != expected {
		t.Fatalf("HekadConfig.Hostname expected: '%s', Got: %s", expected, config.Hostname)
	}
	globals, _, _ := setGlobalConfigs(config)
	if globals.Hostname != expected {
		t.Fatalf("globals.Hostname expected: '%s', Got: %s", expected, globals.Hostname)
	}
	pConfig := pipeline.NewPipelineConfig(globals)
	if pConfig.Hostname() != expected {
		t.Fatalf("PipelineConfig.Hostname expected: '%s', Got: %s", expected, pConfig.Hostname())
	}
	err = loadFullConfig(pConfig, &configPath)
	if err != nil {
		t.Fatalf("Error loading full config: %s", err.Error())
	}
}
Esempio n. 5
0
func main() {
	configFile := flag.String("config", "/etc/hekad.toml", "Config file")
	version := flag.Bool("version", false, "Output version and exit")
	flag.Parse()

	if flag.NFlag() == 0 {
		flag.PrintDefaults()
		os.Exit(0)
	}

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	config, err := LoadHekadConfig(*configFile)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}

	maxprocs := config.Maxprocs
	poolSize := config.PoolSize
	decoderPoolSize := config.DecoderPoolSize
	chanSize := config.ChanSize
	cpuProfName := config.CpuProfName
	memProfName := config.MemProfName
	maxMsgLoops := config.MaxMsgLoops
	maxMsgProcessInject := config.MaxMsgProcessInject
	maxMsgTimerInject := config.MaxMsgTimerInject

	runtime.GOMAXPROCS(maxprocs)

	if cpuProfName != "" {
		profFile, err := os.Create(cpuProfName)
		if err != nil {
			log.Fatalln(err)
		}
		profFile.Close()
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if memProfName != "" {
		defer func() {
			profFile, err := os.Create(memProfName)
			if err != nil {
				log.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Set up and load the pipeline configuration and start the daemon.
	globals := pipeline.DefaultGlobals()
	globals.PoolSize = poolSize
	globals.DecoderPoolSize = decoderPoolSize
	globals.PluginChanSize = chanSize
	globals.MaxMsgLoops = maxMsgLoops
	if globals.MaxMsgLoops == 0 {
		globals.MaxMsgLoops = 1
	}
	globals.MaxMsgProcessInject = maxMsgProcessInject
	globals.MaxMsgTimerInject = maxMsgTimerInject
	pipeconf := pipeline.NewPipelineConfig(globals)

	err = pipeconf.LoadFromConfigFile(*configFile)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}
Esempio n. 6
0
func FilterSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	fth := NewFilterTestHelper(ctrl)
	inChan := make(chan *pipeline.PipelinePack, 1)
	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxFilter", func() {
		sbFilter := new(SandboxFilter)
		sbFilter.SetPipelineConfig(pConfig)
		config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
		config.MemoryLimit = 32000
		config.InstructionLimit = 1000
		config.OutputLimit = 1024

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("Uninitialized", func() {
			err := sbFilter.ReportMsg(msg)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Over inject messages from ProcessMessage", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(2)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(2)

			config.ScriptFilename = "../lua/testsupport/processinject.lua"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			termErr := pipeline.TerminatedError("exceeded InjectMessage count")
			c.Expect(err.Error(), gs.Equals, termErr.Error())
		})

		c.Specify("Over inject messages from TimerEvent", func() {
			var timer <-chan time.Time
			timer = time.Tick(time.Duration(1) * time.Millisecond)
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(11)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(11)

			config.ScriptFilename = "../lua/testsupport/timerinject.lua"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			go func() {
				time.Sleep(time.Duration(250) * time.Millisecond)
				close(inChan)
			}()
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			termErr := pipeline.TerminatedError("exceeded InjectMessage count")
			c.Expect(err.Error(), gs.Equals, termErr.Error())
		})

		c.Specify("Preserves data", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)

			config.ScriptFilename = "../lua/testsupport/serialize.lua"
			config.PreserveData = true
			sbFilter.SetName("serialize")
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			_, err = os.Stat("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
			err = os.Remove("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
		})
	})

	c.Specify("A SandboxManagerFilter", func() {
		pConfig.Globals.BaseDir = os.TempDir()
		sbxMgrsDir := filepath.Join(pConfig.Globals.BaseDir, "sbxmgrs")
		defer func() {
			tmpErr := os.RemoveAll(sbxMgrsDir)
			c.Expect(tmpErr, gs.IsNil)
		}()

		sbmFilter := new(SandboxManagerFilter)
		sbmFilter.SetPipelineConfig(pConfig)
		config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
		config.MaxFilters = 1

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("Control message in the past", func() {
			sbmFilter.Init(config)
			pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: 5 seconds skew"))
			inChan <- pack
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Control message in the future", func() {
			sbmFilter.Init(config)
			pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: -5 seconds skew"))
			inChan <- pack
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Generates the right default working directory", func() {
			sbmFilter.Init(config)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			name := "SandboxManagerFilter"
			fth.MockFilterRunner.EXPECT().Name().Return(name)
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
			_, err = os.Stat(sbxMgrsDir)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Sanity check the default sandbox configuration limits", func() {
			sbmFilter.Init(config)
			c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
			c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
			c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
		})

		c.Specify("Sanity check the user specified sandbox configuration limits", func() {
			config.MemoryLimit = 123456
			config.InstructionLimit = 4321
			config.OutputLimit = 8765
			sbmFilter.Init(config)
			c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
			c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
			c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
		})

		c.Specify("Creates a SandboxFilter runner", func() {
			sbxName := "SandboxFilter"
			sbxMgrName := "SandboxManagerFilter"
			code := `
			require("cjson")

			function process_message()
			    inject_payload(cjson.encode({a = "b"}))
			    return 0
			end
			`
			cfg := `
			[%s]
			type = "SandboxFilter"
			message_matcher = "TRUE"
			script_type = "lua"
			`
			cfg = fmt.Sprintf(cfg, sbxName)
			msg.SetPayload(code)
			f, err := message.NewField("config", cfg, "toml")
			c.Assume(err, gs.IsNil)
			msg.AddField(f)

			fMatchChan := pConfig.Router().AddFilterMatcher()
			errChan := make(chan error)

			fth.MockFilterRunner.EXPECT().Name().Return(sbxMgrName)
			fullSbxName := fmt.Sprintf("%s-%s", sbxMgrName, sbxName)
			fth.MockHelper.EXPECT().Filter(fullSbxName).Return(nil, false)
			fth.MockFilterRunner.EXPECT().LogMessage(fmt.Sprintf("Loading: %s", fullSbxName))

			sbmFilter.Init(config)
			go func() {
				err := sbmFilter.loadSandbox(fth.MockFilterRunner, fth.MockHelper, sbxMgrsDir,
					msg)
				errChan <- err
			}()

			fMatch := <-fMatchChan
			c.Expect(fMatch.MatcherSpecification().String(), gs.Equals, "TRUE")
			c.Expect(<-errChan, gs.IsNil)

			go func() {
				<-pConfig.Router().RemoveFilterMatcher()
			}()
			ok := pConfig.RemoveFilterRunner(fullSbxName)
			c.Expect(ok, gs.IsTrue)
		})
	})

	c.Specify("A Cpu Stats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "cpustats"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/cpustats.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		msg := getTestMessage()
		fields := make([]*message.Field, 4)
		fields[0], _ = message.NewField("1MinAvg", 0.08, "")
		fields[1], _ = message.NewField("5MinAvg", 0.04, "")
		fields[2], _ = message.NewField("15MinAvg", 0.02, "")
		fields[3], _ = message.NewField("NumProcesses", 5, "")
		msg.Fields = fields

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().Name().Return("cpustats")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with cpuload data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			for i := 1; i <= 3; i++ {
				// Fill in the data
				t := int64(i * 1000000000)
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":1,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"1MinAvg","unit":"Count","aggregation":"max"},{"name":"5MinAvg","unit":"Count","aggregation":"max"},{"name":"15MinAvg","unit":"Count","aggregation":"max"},{"name":"NumProcesses","unit":"Count","aggregation":"max"}]}
0.08	0.04	0.02	5
0.08	0.04	0.02	5
0.08	0.04	0.02	5
`

			c.Expect(p.Message.GetPayload(), gs.Equals, pl)
		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("A Memstats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "memstats"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/memstats.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		msg := getTestMessage()
		field_names := []string{"MemFree", "Cached", "Active", "Inactive", "VmallocUsed", "Shmem", "SwapCached", "SwapTotal", "SwapFree"}
		fields := make([]*message.Field, len(field_names))
		for i, name := range field_names {
			fields[i], _ = message.NewField(name, 100, "")
		}
		msg.Fields = fields

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().Name().Return("memstats")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with memstats data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			for i := 1; i <= 3; i++ {
				// Fill in the data
				t := int64(i * 1000000000)
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":1,"rows":3,"columns":9,"seconds_per_row":1,"column_info":[{"name":"MemFree","unit":"Count","aggregation":"max"},{"name":"Cached","unit":"Count","aggregation":"max"},{"name":"Active","unit":"Count","aggregation":"max"},{"name":"Inactive","unit":"Count","aggregation":"max"},{"name":"VmallocUsed","unit":"Count","aggregation":"max"},{"name":"Shmem","unit":"Count","aggregation":"max"},{"name":"SwapCached","unit":"Count","aggregation":"max"},{"name":"SwapFree","unit":"Count","aggregation":"max"},{"name":"SwapUsed","unit":"Count","aggregation":"max"}]}
100	100	100	100	100	100	100	100	0
100	100	100	100	100	100	100	100	0
100	100	100	100	100	100	100	100	0
`
			c.Expect(p.Message.GetPayload(), gs.Equals, pl)

		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("A diskstats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "diskstats"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/diskstats.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retMsgChan := make(chan *message.Message, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retMsgChan)
		}()

		msg := getTestMessage()

		field_names := []string{
			"WritesCompleted",
			"ReadsCompleted",
			"SectorsWritten",
			"SectorsRead",
			"WritesMerged",
			"ReadsMerged",
			"TimeWriting",
			"TimeReading",
			"TimeDoingIO",
			"WeightedTimeDoingIO",
		}

		num_fields := len(field_names) + 1
		fields := make([]*message.Field, num_fields)
		msg.Fields = fields
		timeInterval, _ := message.NewField("TickerInterval", 1, "")
		fields[num_fields-1] = timeInterval
		fieldVal := 100

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelineConfig().AnyTimes()
		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).AnyTimes()
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer).AnyTimes()
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan).AnyTimes()
		fth.MockFilterRunner.EXPECT().Name().Return("diskstats").AnyTimes()
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			msg := pack.Message
			pack.Message = new(message.Message)
			retMsgChan <- msg
		}).Return(true).AnyTimes()

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with diskstats data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			// Iterate 4 times since the first one doesn't actually set the cbuf
			// in order to set the delta in the cbuf
			for i := 1; i <= 4; i++ {
				// Fill in the fields
				for i, name := range field_names {
					fields[i], _ = message.NewField(name, fieldVal, "")
				}
				// Scale up the value so we can see the delta growing
				// by 100 each iteration
				fieldVal += i * 100

				t := int64(i * 1000000000)
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			testExpects := map[string]string{
				"Time doing IO": `{"time":2,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"TimeWriting","unit":"ms","aggregation":"max"},{"name":"TimeReading","unit":"ms","aggregation":"max"},{"name":"TimeDoingIO","unit":"ms","aggregation":"max"},{"name":"WeightedTimeDoi","unit":"ms","aggregation":"max"}]}
200	200	200	200
400	400	400	400
700	700	700	700
`,
				"Disk Stats": `{"time":2,"rows":3,"columns":6,"seconds_per_row":1,"column_info":[{"name":"WritesCompleted","unit":"per_1_s","aggregation":"none"},{"name":"ReadsCompleted","unit":"per_1_s","aggregation":"none"},{"name":"SectorsWritten","unit":"per_1_s","aggregation":"none"},{"name":"SectorsRead","unit":"per_1_s","aggregation":"none"},{"name":"WritesMerged","unit":"per_1_s","aggregation":"none"},{"name":"ReadsMerged","unit":"per_1_s","aggregation":"none"}]}
100	100	100	100	100	100
200	200	200	200	200	200
300	300	300	300	300	300
`,
			}

			timer <- time.Now()
			for i := 0; i < 2; i++ {
				m := <-retMsgChan
				name, ok := m.GetFieldValue("payload_name")
				c.Assume(ok, gs.IsTrue)
				nameVal, ok := name.(string)
				c.Assume(ok, gs.IsTrue)
				c.Expect(m.GetPayload(), gs.Equals, testExpects[nameVal])
			}
		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("http_status filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "http_status"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/http_status.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(2)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		field, _ := message.NewField("status", 0, "")
		msg := &message.Message{}
		msg.SetTimestamp(0)
		msg.AddField(field)

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().Name().Return("http_status")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with http status data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			for i := 0; i <= 6; i++ {
				msg.Fields[0].ValueInteger[0] = int64(i * 100) // iterate through the status codes with a bogus status on each end
				pack.Message = msg
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":0,"rows":2,"columns":6,"seconds_per_row":1,"column_info":[{"name":"HTTP_100","unit":"count","aggregation":"sum"},{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1	1	1	1	1	2
nan	nan	nan	nan	nan	nan
`
			c.Expect(p.Message.GetPayload(), gs.Equals, pl)

		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})
}
Esempio n. 7
0
func DashboardOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := pipeline.NewPipelineConfig(nil)

	dashboardOutput := new(DashboardOutput)
	dashboardOutput.pConfig = pConfig

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)

	errChan := make(chan error, 1)

	startOutput := func() {
		go func() {
			errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
		}()
	}

	if runtime.GOOS != "windows" {
		c.Specify("A DashboardOutput", func() {

			tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
			c.Assume(err, gs.IsNil)
			config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
			config.WorkingDirectory = tmpdir

			c.Specify("Init halts if basedirectory is not writable", func() {
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)
				err = dashboardOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
			})

			c.Specify("that is running", func() {
				startedChan := make(chan bool, 1)
				defer close(startedChan)
				ts := httptest.NewUnstartedServer(nil)

				dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
					ts.Start()
					startedChan <- true
					return nil
				}

				ticker := make(chan time.Time)
				inChan := make(chan *pipeline.PipelinePack, 1)
				recycleChan := make(chan *pipeline.PipelinePack, 1)
				pack := pipeline.NewPipelinePack(recycleChan)
				pack.Message = pipeline_ts.GetTestMessage()

				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)

				err := os.MkdirAll(tmpdir, 0700)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)

				dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					// Noop
				})

				c.Specify("sets custom http headers", func() {
					config.Headers = http.Header{
						"One":  []string{"two", "three"},
						"Four": []string{"five", "six", "seven"},
					}
					err = dashboardOutput.Init(config)
					c.Assume(err, gs.IsNil)
					ts.Config = dashboardOutput.server

					startOutput()

					inChan <- pack
					<-startedChan
					resp, err := http.Get(ts.URL)
					c.Assume(err, gs.IsNil)
					resp.Body.Close()
					c.Assume(resp.StatusCode, gs.Equals, 200)

					// Verify headers are there
					eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
					c.Expect(eq, gs.IsTrue)
					eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
					c.Expect(eq, gs.IsTrue)
				})

				close(inChan)
				c.Expect(<-errChan, gs.IsNil)

				ts.Close()
			})
		})
	}
}
Esempio n. 8
0
func EncoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// NewPipelineConfig sets up Globals which is needed for the
	// pipeline.Prepend*Dir functions to not die during plugin Init().
	_ = pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxEncoder", func() {

		encoder := new(SandboxEncoder)
		conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message.SetPayload("original")
		pack.Message.SetType("my_type")
		pack.Message.SetPid(12345)
		pack.Message.SetSeverity(4)
		pack.Message.SetHostname("hostname")
		pack.Message.SetTimestamp(54321)
		pack.Message.SetUuid(uuid.NewRandom())
		var (
			result []byte
			err    error
		)

		c.Specify("emits JSON correctly", func() {
			conf.ScriptFilename = "../lua/testsupport/encoder_json.lua"
			err = encoder.Init(conf)
			c.Expect(err, gs.IsNil)

			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			msg := new(message.Message)
			err = json.Unmarshal(result, msg)
			c.Expect(err, gs.IsNil)
			c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
			c.Expect(msg.GetPid(), gs.Equals, int32(12345))
			c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
			c.Expect(msg.GetHostname(), gs.Equals, "hostname")
			c.Expect(msg.GetPayload(), gs.Equals, "original")
			c.Expect(msg.GetType(), gs.Equals, "my_type")
		})

		c.Specify("emits text correctly", func() {
			conf.ScriptFilename = "../lua/testsupport/encoder_text.lua"
			err = encoder.Init(conf)
			c.Expect(err, gs.IsNil)

			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(string(result), gs.Equals, "Prefixed original")
		})

		c.Specify("emits protobuf correctly", func() {

			c.Specify("when inject_message is used", func() {
				conf.ScriptFilename = "../lua/testsupport/encoder_protobuf.lua"
				err = encoder.Init(conf)
				c.Expect(err, gs.IsNil)

				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				msg := new(message.Message)
				err = proto.Unmarshal(result, msg)
				c.Expect(err, gs.IsNil)
				c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
				c.Expect(msg.GetPid(), gs.Equals, int32(12345))
				c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
				c.Expect(msg.GetHostname(), gs.Equals, "hostname")
				c.Expect(msg.GetPayload(), gs.Equals, "mutated")
				c.Expect(msg.GetType(), gs.Equals, "after")
			})

			c.Specify("when `write_message` is used", func() {
				conf.ScriptFilename = "../lua/testsupport/encoder_writemessage.lua"
				err = encoder.Init(conf)
				c.Expect(err, gs.IsNil)

				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				msg := new(message.Message)
				err = proto.Unmarshal(result, msg)
				c.Expect(err, gs.IsNil)
				c.Expect(msg.GetPayload(), gs.Equals, "mutated payload")
				c.Expect(pack.Message.GetPayload(), gs.Equals, "original")
			})
		})

	})
}
Esempio n. 9
0
func FilterSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	fth := NewFilterTestHelper(ctrl)
	inChan := make(chan *pipeline.PipelinePack, 1)
	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxFilter", func() {
		sbFilter := new(SandboxFilter)
		sbFilter.SetPipelineConfig(pConfig)
		config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
		config.MemoryLimit = 64000
		config.InstructionLimit = 1000
		config.OutputLimit = 1024

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
		pack.Message = msg

		c.Specify("Uninitialized", func() {
			err := sbFilter.ReportMsg(msg)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Over inject messages from ProcessMessage", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
			fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(2)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(2)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

			config.ScriptFilename = "../lua/testsupport/processinject.lua"
			config.ModuleDirectory = "../lua/modules"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			var termErr pipeline.TerminatedError
			if runtime.GOOS == "windows" {
				termErr = pipeline.TerminatedError("process_message() ..\\lua\\testsupport\\processinject.lua:8: inject_payload() exceeded InjectMessage count")
			} else {
				termErr = pipeline.TerminatedError("process_message() ../lua/testsupport/processinject.lua:8: inject_payload() exceeded InjectMessage count")
			}
			c.Expect(err.Error(), gs.Equals, termErr.Error())
		})

		c.Specify("Over inject messages from TimerEvent", func() {
			var timer <-chan time.Time
			timer = time.Tick(time.Duration(1) * time.Millisecond)
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
			fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(11)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(11)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

			config.ScriptFilename = "../lua/testsupport/timerinject.lua"
			config.ModuleDirectory = "../lua/modules"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			go func() {
				time.Sleep(time.Duration(250) * time.Millisecond)
				close(inChan)
			}()
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)

			var termErr pipeline.TerminatedError
			if runtime.GOOS == "windows" {
				termErr = pipeline.TerminatedError("timer_event() ..\\lua\\testsupport\\timerinject.lua:13: inject_payload() exceeded InjectMessage count")
			} else {
				termErr = pipeline.TerminatedError("timer_event() ../lua/testsupport/timerinject.lua:13: inject_payload() exceeded InjectMessage count")
			}
			c.Expect(err.Error(), gs.Equals, termErr.Error())
		})

		c.Specify("Preserves data", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

			config.ScriptFilename = "../lua/testsupport/serialize.lua"
			config.ModuleDirectory = "../lua/modules"
			config.PreserveData = true
			sbFilter.SetName("serialize")
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			_, err = os.Stat("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
			err = os.Remove("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
		})

		c.Specify("process_message error string", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("script provided error message"))
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

			config.ScriptFilename = "../lua/testsupport/process_message_error_string.lua"
			config.ModuleDirectory = "../lua/modules"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Over inject messages from the shutdown TimerEvent", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
			fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(10)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(10)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(10)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

			config.ScriptFilename = "../lua/testsupport/timerinject.lua"
			config.ModuleDirectory = "../lua/modules"
			config.TimerEventOnShutdown = true
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			if runtime.GOOS == "windows" {
				c.Expect(err.Error(), gs.Equals, "FATAL: timer_event() ..\\lua\\testsupport\\timerinject.lua:13: inject_payload() exceeded InjectMessage count")
			} else {
				c.Expect(err.Error(), gs.Equals, "FATAL: timer_event() ../lua/testsupport/timerinject.lua:13: inject_payload() exceeded InjectMessage count")
			}
		})
	})

	c.Specify("A SandboxManagerFilter", func() {
		pConfig.Globals.BaseDir = os.TempDir()
		sbxMgrsDir := filepath.Join(pConfig.Globals.BaseDir, "sbxmgrs")
		defer func() {
			tmpErr := os.RemoveAll(sbxMgrsDir)
			c.Expect(tmpErr, gs.IsNil)
		}()

		sbmFilter := new(SandboxManagerFilter)
		sbmFilter.SetPipelineConfig(pConfig)
		config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
		config.MaxFilters = 1

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg

		c.Specify("Control message in the past", func() {
			sbmFilter.Init(config)
			pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UpdateCursor("")
			fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
			pack.BufferedPack = true
			pack.DelivErrChan = make(chan error, 1)
			inChan <- pack
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			err = <-pack.DelivErrChan
			c.Expect(err.Error(), gs.Equals, "Discarded control message: 5 seconds skew")
		})

		c.Specify("Control message in the future", func() {
			sbmFilter.Init(config)
			pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UpdateCursor("")
			fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
			pack.BufferedPack = true
			pack.DelivErrChan = make(chan error, 1)
			inChan <- pack
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			err = <-pack.DelivErrChan
			c.Expect(err.Error(), gs.Equals, "Discarded control message: -5 seconds skew")
		})

		c.Specify("Generates the right default working directory", func() {
			sbmFilter.Init(config)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			name := "SandboxManagerFilter"
			fth.MockFilterRunner.EXPECT().Name().Return(name)
			close(inChan)
			err := sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
			_, err = os.Stat(sbxMgrsDir)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Sanity check the default sandbox configuration limits", func() {
			sbmFilter.Init(config)
			c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
			c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
			c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
		})

		c.Specify("Sanity check the user specified sandbox configuration limits", func() {
			config.MemoryLimit = 123456
			config.InstructionLimit = 4321
			config.OutputLimit = 8765
			sbmFilter.Init(config)
			c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
			c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
			c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
		})

		c.Specify("Creates a SandboxFilter runner", func() {
			sbxName := "SandboxFilter"
			sbxMgrName := "SandboxManagerFilter"
			code := `

			function process_message()
			    inject_payload('{a = "b"}')
			    return 0
			end
			`
			cfg := `
			[%s]
			type = "SandboxFilter"
			message_matcher = "TRUE"
			script_type = "lua"
			`
			cfg = fmt.Sprintf(cfg, sbxName)
			msg.SetPayload(code)
			f, err := message.NewField("config", cfg, "toml")
			c.Assume(err, gs.IsNil)
			msg.AddField(f)

			fMatchChan := pConfig.Router().AddFilterMatcher()
			errChan := make(chan error)

			fth.MockFilterRunner.EXPECT().Name().Return(sbxMgrName)
			fullSbxName := fmt.Sprintf("%s-%s", sbxMgrName, sbxName)
			fth.MockHelper.EXPECT().Filter(fullSbxName).Return(nil, false)
			fth.MockFilterRunner.EXPECT().LogMessage(fmt.Sprintf("Loading: %s", fullSbxName))

			sbmFilter.Init(config)
			go func() {
				err := sbmFilter.loadSandbox(fth.MockFilterRunner, fth.MockHelper, sbxMgrsDir,
					msg)
				errChan <- err
			}()

			fMatch := <-fMatchChan
			c.Expect(fMatch.MatcherSpecification().String(), gs.Equals, "TRUE")
			c.Expect(<-errChan, gs.IsNil)

			go func() {
				<-pConfig.Router().RemoveFilterMatcher()
			}()
			ok := pConfig.RemoveFilterRunner(fullSbxName)
			c.Expect(ok, gs.IsTrue)
		})
	})

	c.Specify("A Load Average Stats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "loadavg"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/loadavg.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		msg := getTestMessage()
		fields := make([]*message.Field, 4)
		fields[0], _ = message.NewField("1MinAvg", 0.08, "")
		fields[1], _ = message.NewField("5MinAvg", 0.04, "")
		fields[2], _ = message.NewField("15MinAvg", 0.02, "")
		fields[3], _ = message.NewField("NumProcesses", 5, "")
		msg.Fields = fields

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
		fth.MockFilterRunner.EXPECT().Name().Return("loadavg")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with loadavg data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			var i int64
			for i = 1; i <= 3; i++ {
				// Fill in the data
				t := i * 1000000000
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":1,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"1MinAvg","unit":"Count","aggregation":"max"},{"name":"5MinAvg","unit":"Count","aggregation":"max"},{"name":"15MinAvg","unit":"Count","aggregation":"max"},{"name":"NumProcesses","unit":"Count","aggregation":"max"}]}
0.08	0.04	0.02	5
0.08	0.04	0.02	5
0.08	0.04	0.02	5
`

			c.Expect(p.Message.GetPayload(), gs.Equals, pl)
		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("A Memstats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "memstats"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/memstats.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		msg := getTestMessage()
		field_names := []string{"MemFree", "Cached", "Active", "Inactive", "VmallocUsed", "Shmem", "SwapCached", "SwapTotal", "SwapFree"}
		fields := make([]*message.Field, len(field_names))
		for i, name := range field_names {
			fields[i], _ = message.NewField(name, 100, "")
		}
		msg.Fields = fields

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
		fth.MockFilterRunner.EXPECT().Name().Return("memstats")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with memstats data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			var i int64
			for i = 1; i <= 3; i++ {
				// Fill in the data
				t := i * 1000000000
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":1,"rows":3,"columns":9,"seconds_per_row":1,"column_info":[{"name":"MemFree","unit":"Count","aggregation":"max"},{"name":"Cached","unit":"Count","aggregation":"max"},{"name":"Active","unit":"Count","aggregation":"max"},{"name":"Inactive","unit":"Count","aggregation":"max"},{"name":"VmallocUsed","unit":"Count","aggregation":"max"},{"name":"Shmem","unit":"Count","aggregation":"max"},{"name":"SwapCached","unit":"Count","aggregation":"max"},{"name":"SwapFree","unit":"Count","aggregation":"max"},{"name":"SwapUsed","unit":"Count","aggregation":"max"}]}
100	100	100	100	100	100	100	100	0
100	100	100	100	100	100	100	100	0
100	100	100	100	100	100	100	100	0
`
			c.Expect(p.Message.GetPayload(), gs.Equals, pl)

		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("A diskstats filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "diskstats"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/diskstats.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(3)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retMsgChan := make(chan *message.Message, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retMsgChan)
		}()

		msg := getTestMessage()

		field_names := []string{
			"WritesCompleted",
			"ReadsCompleted",
			"SectorsWritten",
			"SectorsRead",
			"WritesMerged",
			"ReadsMerged",
			"TimeWriting",
			"TimeReading",
			"TimeDoingIO",
			"WeightedTimeDoingIO",
		}

		num_fields := len(field_names) + 1
		fields := make([]*message.Field, num_fields)
		msg.Fields = fields
		timeInterval, _ := message.NewField("TickerInterval", 1, "")
		fields[num_fields-1] = timeInterval
		var fieldVal int64 = 100

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelineConfig().AnyTimes()
		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).AnyTimes()
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer).AnyTimes()
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan).AnyTimes()
		fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
		fth.MockFilterRunner.EXPECT().Name().Return("diskstats").AnyTimes()
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			msg := pack.Message
			pack.Message = new(message.Message)
			retMsgChan <- msg
		}).Return(true).AnyTimes()

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with diskstats data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			// Iterate 4 times since the first one doesn't actually set the cbuf
			// in order to set the delta in the cbuf
			var i int64
			for i = 1; i <= 4; i++ {
				// Fill in the fields
				for i, name := range field_names {
					fields[i], _ = message.NewField(name, fieldVal, "")
				}
				// Scale up the value so we can see the delta growing
				// by 100 each iteration
				fieldVal += i * 100

				t := i * 1000000000
				pack.Message = msg
				pack.Message.SetTimestamp(t)

				// Feed in a pack
				inChan <- pack
				pack = <-recycleChan
			}

			testExpects := map[string]string{
				"Time doing IO": `{"time":2,"rows":3,"columns":4,"seconds_per_row":1,"column_info":[{"name":"TimeWriting","unit":"ms","aggregation":"max"},{"name":"TimeReading","unit":"ms","aggregation":"max"},{"name":"TimeDoingIO","unit":"ms","aggregation":"max"},{"name":"WeightedTimeDoi","unit":"ms","aggregation":"max"}]}
200	200	200	200
400	400	400	400
700	700	700	700
`,
				"Disk Stats": `{"time":2,"rows":3,"columns":6,"seconds_per_row":1,"column_info":[{"name":"WritesCompleted","unit":"per_1_s","aggregation":"none"},{"name":"ReadsCompleted","unit":"per_1_s","aggregation":"none"},{"name":"SectorsWritten","unit":"per_1_s","aggregation":"none"},{"name":"SectorsRead","unit":"per_1_s","aggregation":"none"},{"name":"WritesMerged","unit":"per_1_s","aggregation":"none"},{"name":"ReadsMerged","unit":"per_1_s","aggregation":"none"}]}
100	100	100	100	100	100
200	200	200	200	200	200
300	300	300	300	300	300
`,
			}

			timer <- time.Now()
			for i := 0; i < 2; i++ {
				m := <-retMsgChan
				name, ok := m.GetFieldValue("payload_name")
				c.Assume(ok, gs.IsTrue)
				nameVal, ok := name.(string)
				c.Assume(ok, gs.IsTrue)
				c.Expect(m.GetPayload(), gs.Equals, testExpects[nameVal])
			}
		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("http_status filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "http_status"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/filters/http_status.lua"
		conf.ModuleDirectory = "../lua/modules"

		conf.Config = make(map[string]interface{})
		conf.Config["rows"] = int64(2)
		conf.Config["sec_per_row"] = int64(1)

		timer := make(chan time.Time, 1)
		errChan := make(chan error, 1)
		retPackChan := make(chan *pipeline.PipelinePack, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retPackChan)
		}()

		field, _ := message.NewField("status", 0, "")
		msg := &message.Message{}
		msg.SetTimestamp(0)
		msg.AddField(field)

		pack := pipeline.NewPipelinePack(recycleChan)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
		fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
		fth.MockFilterRunner.EXPECT().Name().Return("http_status")
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retPackChan <- pack
		}).Return(true)

		err := filter.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("should fill a cbuf with http status data", func() {
			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			for i := 0; i <= 6; i++ {
				msg.Fields[0].ValueInteger[0] = int64(i * 100) // iterate through the status codes with a bogus status on each end
				pack.Message = msg
				inChan <- pack
				pack = <-recycleChan
			}

			timer <- time.Now()
			p := <-retPackChan
			// Check the result of the filter's inject
			pl := `{"time":0,"rows":2,"columns":6,"seconds_per_row":1,"column_info":[{"name":"HTTP_100","unit":"count","aggregation":"sum"},{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1	1	1	1	1	2
nan	nan	nan	nan	nan	nan
`
			c.Expect(p.Message.GetPayload(), gs.Equals, pl)

		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})

	c.Specify("influx_batch filter", func() {
		filter := new(SandboxFilter)
		filter.SetPipelineConfig(pConfig)
		filter.name = "influx_batch"
		conf := filter.ConfigStruct().(*sandbox.SandboxConfig)

		errChan := make(chan error, 1)
		retMsgChan := make(chan *message.Message, 1)
		recycleChan := make(chan *pipeline.PipelinePack, 1)

		defer func() {
			close(errChan)
			close(retMsgChan)
		}()

		pack := pipeline.NewPipelinePack(recycleChan)
		pack.Message = &message.Message{}
		pack.Message.SetType("my_type")
		pack.Message.SetPid(12345)
		pack.Message.SetSeverity(4)
		pack.Message.SetHostname("hostname")
		pack.Message.SetTimestamp(54321 * 1e9)
		pack.Message.SetLogger("Logger")
		pack.Message.SetPayload("Payload value lorem ipsum")
		pack.Message.SetUuid(uuid.Parse("8e414f01-9d7f-4a48-a5e1-ae92e5954df5"))

		f, err := message.NewField("intField", 123, "") // Overwritten in a loop below.
		c.Assume(err, gs.IsNil)
		err = f.AddValue(456)
		c.Assume(err, gs.IsNil)
		pack.Message.AddField(f)

		f, err = message.NewField("strField", "0_first", "")
		c.Assume(err, gs.IsNil)
		err = f.AddValue("0_second")
		c.Assume(err, gs.IsNil)
		pack.Message.AddField(f)

		f, err = message.NewField("strField", "1_first", "")
		c.Assume(err, gs.IsNil)
		err = f.AddValue("1_second")
		c.Assume(err, gs.IsNil)
		pack.Message.AddField(f)

		f, err = message.NewField("byteField", []byte("first"), "")
		c.Assume(err, gs.IsNil)
		err = f.AddValue([]byte("second"))
		c.Assume(err, gs.IsNil)
		pack.Message.AddField(f)

		fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil)
		fth.MockFilterRunner.EXPECT().Ticker().Return(nil)
		fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
		fth.MockFilterRunner.EXPECT().UsesBuffering().Return(false)
		fth.MockFilterRunner.EXPECT().Name().Return(filter.name)
		fth.MockFilterRunner.EXPECT().Inject(pack).Do(func(pack *pipeline.PipelinePack) {
			retMsgChan <- pack.Message
		}).Return(true)

		c.Assume(err, gs.IsNil)

		conf.ScriptFilename = "../lua/filters/influx_batch.lua"
		conf.ModuleDirectory = "../lua/modules"
		conf.Config = make(map[string]interface{})

		c.Specify("encodes a basic message", func() {
			conf.Config["flush_count"] = "6"
			err := filter.Init(conf)
			c.Assume(err, gs.IsNil)

			go func() {
				errChan <- filter.Run(fth.MockFilterRunner, fth.MockHelper)
			}()

			// Having our own reference to the message struct lets us hand it
			// back to the pack even after recycling.
			msg := pack.Message
			for i := 0; i < 6; i++ {
				msg.Fields[0].ValueInteger[0] = int64(i * 100)
				pack.Message = msg
				pack.EncodeMsgBytes()
				inChan <- pack
				pack = <-recycleChan
			}

			// timer <- time.Now()
			m := <-retMsgChan
			// Check the result of the filter's inject
			msgStr := `byteField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="first" 54321000
byteField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="second" 54321000
strField_fidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="1_first" 54321000
strField_fidx_1_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="1_second" 54321000
strField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="0_first" 54321000
strField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value="0_second" 54321000
intField,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value=%d.000000 54321000
intField_vidx_1,Logger=Logger,Type=my_type,Severity=4,Hostname=hostname value=456.000000 54321000
`
			pl := ""
			for i := 0; i < 6; i++ {
				pl += fmt.Sprintf(msgStr, i*100)
			}
			c.Expect(m.GetPayload(), gs.Equals, pl)

		})

		close(inChan)
		c.Expect(<-errChan, gs.IsNil)
	})
}
Esempio n. 10
0
func DecoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// NewPipelineConfig sets up Globals which is needed for the
	// pipeline.Prepend*Dir functions to not die during plugin Init().
	_ = pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxDecoder", func() {

		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		conf.ScriptType = "lua"

		c.Specify("that uses lpeg and inject_message", func() {
			dRunner.EXPECT().Name().Return("serialize")
			conf.ScriptFilename = "../lua/testsupport/decoder.lua"
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			c.Specify("decodes simple messages", func() {
				data := "1376389920 debug id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				_, err = decoder.Decode(pack)
				c.Assume(err, gs.IsNil)

				c.Expect(pack.Message.GetTimestamp(),
					gs.Equals,
					int64(1376389920000000000))

				c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

				var ok bool
				var value interface{}
				value, ok = pack.Message.GetFieldValue("id")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "2321")

				value, ok = pack.Message.GetFieldValue("url")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "example.com")

				value, ok = pack.Message.GetFieldValue("item")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "1")
				decoder.Shutdown()
			})

			c.Specify("decodes an invalid messages", func() {
				data := "1376389920 bogus id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(len(packs), gs.Equals, 0)
				c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
				c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
				decoder.Shutdown()
			})

			c.Specify("Preserves data", func() {
				conf.ScriptFilename = "../lua/testsupport/serialize.lua"
				conf.PreserveData = true
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)
				decoder.Shutdown()
				_, err = os.Stat("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
				err = os.Remove("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("that only uses write_message", func() {
			conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
			dRunner.EXPECT().Name().Return("write_message")
			err := decoder.Init(conf)
			decoder.SetDecoderRunner(dRunner)
			c.Assume(err, gs.IsNil)

			c.Specify("adds a string field to the message", func() {
				data := "string field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(string), gs.Equals, "foo")
			})

			c.Specify("adds a numeric field to the message", func() {
				data := "num field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(float64), gs.Equals, float64(1))
			})

			c.Specify("adds a boolean field to the message", func() {
				data := "bool field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(bool), gs.Equals, true)
			})

			c.Specify("sets type and payload", func() {
				data := "set type and payload"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
				c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
			})

			c.Specify("sets field value with representation", func() {
				data := "set field value with representation"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("rep")
				c.Expect(len(fields), gs.Equals, 1)
				field := fields[0]
				values := field.GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "foo")
				c.Expect(field.GetRepresentation(), gs.Equals, "representation")
			})

			c.Specify("sets multiple field string values", func() {
				data := "set multiple field string values"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("multi")
				c.Expect(len(fields), gs.Equals, 2)
				values := fields[0].GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "first")
				values = fields[1].GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "second")
			})

			c.Specify("sets field string array value", func() {
				data := "set field string array value"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("array")
				c.Expect(len(fields), gs.Equals, 1)
				values := fields[0].GetValueString()
				c.Expect(len(values), gs.Equals, 2)
				c.Expect(values[0], gs.Equals, "first")
				c.Expect(values[1], gs.Equals, "second")
			})
		})
	})

	c.Specify("A Multipack SandboxDecoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
		conf.ScriptType = "lua"
		supply := make(chan *pipeline.PipelinePack, 3)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message = getTestMessage()

		pack1 := pipeline.NewPipelinePack(supply)
		pack2 := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")

		c.Specify("decodes into multiple packs", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			decoder.SetDecoderRunner(dRunner)
			gomock.InOrder(
				dRunner.EXPECT().NewPack().Return(pack1),
				dRunner.EXPECT().NewPack().Return(pack2),
			)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 3)
			c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
			c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
			c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")

			for i := 0; i < 1; i++ {
				c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
				c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
				c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
				c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))

			}
			decoder.Shutdown()
		})
	})

	c.Specify("Nginx access log decoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.ScriptType = "lua"
		conf.Config = make(map[string]interface{})
		conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
		conf.Config["user_agent_transform"] = true
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392050801000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("remote_addr")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "127.0.0.1")

			value, ok = pack.Message.GetFieldValue("user_agent_browser")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Firefox")
			value, ok = pack.Message.GetFieldValue("user_agent_version")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(26))
			value, ok = pack.Message.GetFieldValue("user_agent_os")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Linux")
			_, ok = pack.Message.GetFieldValue("http_user_agent")
			c.Expect(ok, gs.Equals, false)

			value, ok = pack.Message.GetFieldValue("body_bytes_sent")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("status")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(304))
			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("rsyslog decoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.ScriptType = "lua"
		conf.Config = make(map[string]interface{})
		conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
		conf.Config["tz"] = "America/Los_Angeles"
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost kernel: imklog 5.8.6, log source = /proc/kmsg started.\n"
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392065938000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
			c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
			c.Expect(pack.Message.GetPayload(), gs.Equals, "imklog 5.8.6, log source = /proc/kmsg started.")

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("syslogtag")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "kernel:")

			value, ok = pack.Message.GetFieldValue("syslogfacility")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(3))

			value, ok = pack.Message.GetFieldValue("timegenerated")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(1392065939000000000))

			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})
}
func HttpInputSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)

	jsonPost := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`

	c.Specify("A HttpInput", func() {

		httpInput := pipeline.HttpInput{}

		c.Specify("honors time ticker to flush", func() {
			ith := new(pipeline.InputTestHelper)
			ith.MockHelper = NewMockPluginHelper(ctrl)
			ith.MockInputRunner = NewMockInputRunner(ctrl)
			startInput := func() {
				go func() {
					err := httpInput.Run(ith.MockInputRunner, ith.MockHelper)
					c.Expect(err, gs.IsNil)
				}()
			}

			ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
			ith.PackSupply = make(chan *PipelinePack, 1)
			ith.PackSupply <- ith.Pack

			ith.Decoders = make([]DecoderRunner, int(message.Header_JSON+1))
			ith.Decoders[message.Header_JSON] = NewMockDecoderRunner(ctrl)
			ith.MockDecoderSet = NewMockDecoderSet(ctrl)

			// Spin up a http server
			server, err := ts.NewOneHttpServer(jsonPost, "localhost", 9876)
			c.Expect(err, gs.IsNil)
			go server.Start("/")
			time.Sleep(10 * time.Millisecond)

			config := httpInput.ConfigStruct().(*HttpInputConfig)
			config.DecoderName = "JsonDecoder"
			config.Url = "http://localhost:9876/"
			tickChan := make(chan time.Time)

			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
			ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)

			ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
			ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

			mockDecoderRunner := ith.Decoders[message.Header_JSON].(*MockDecoderRunner)

			// Stub out the DecoderRunner input channel so that we can
			// inspect bytes later on
			dRunnerInChan := make(chan *PipelinePack, 1)
			mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)

			dset := ith.MockDecoderSet.EXPECT().ByName("JsonDecoder")
			dset.Return(ith.Decoders[message.Header_JSON], true)

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()

			tickChan <- time.Now()

			// We need for the pipeline to finish up
			time.Sleep(50 * time.Millisecond)
		})

		c.Specify("short circuits packs into the router", func() {
			ith := new(InputTestHelper)
			ith.MockHelper = NewMockPluginHelper(ctrl)
			ith.MockInputRunner = NewMockInputRunner(ctrl)
			startInput := func() {
				go func() {
					err := httpInput.Run(ith.MockInputRunner, ith.MockHelper)
					c.Expect(err, gs.IsNil)
				}()
			}
			ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
			ith.PackSupply = make(chan *PipelinePack, 1)
			ith.PackSupply <- ith.Pack

			ith.Decoders = make([]DecoderRunner, int(message.Header_JSON+1))
			ith.Decoders[message.Header_JSON] = NewMockDecoderRunner(ctrl)
			ith.MockDecoderSet = NewMockDecoderSet(ctrl)
			config := httpInput.ConfigStruct().(*HttpInputConfig)
			config.Url = "http://localhost:9876/"
			tickChan := make(chan time.Time)

			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
			ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)

			ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
			ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

			err := httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()

			tickChan <- time.Now()

			// We need for the pipeline to finish up
			time.Sleep(50 * time.Millisecond)
		})

	})
}
Esempio n. 12
0
func TestWriteMessage(t *testing.T) {
	pipeline.NewPipelineConfig(nil) // Set up globals.
	var sbc SandboxConfig
	sbc.ScriptFilename = "./testsupport/field_scribble.lua"
	sbc.MemoryLimit = 32767
	sbc.InstructionLimit = 1000
	pack := getTestPack()
	sb, err := lua.CreateLuaSandbox(&sbc)
	if err != nil {
		t.Errorf("%s", err)
	}
	err = sb.Init("", "decoder")
	if err != nil {
		t.Errorf("%s", err)
	}
	r := sb.ProcessMessage(pack)
	if r != 0 {
		t.Errorf("ProcessMessage should return 0, received %d", r)
	}
	if pack.Message.GetType() != "MyType" {
		t.Error("Type not set")
	}
	if pack.Message.GetLogger() != "MyLogger" {
		t.Error("Logger not set")
	}
	packTime := time.Unix(0, pack.Message.GetTimestamp())
	cmpTime := time.Unix(0, 1385968914904958136)
	d, _ := time.ParseDuration("500ns")
	packTime = packTime.Round(d)
	cmpTime = cmpTime.Round(d)
	if !packTime.Equal(cmpTime) {
		t.Errorf("Timestamp not set: %d", pack.Message.GetTimestamp())
	}
	if pack.Message.GetPayload() != "MyPayload" {
		t.Error("Payload not set")
	}
	if pack.Message.GetEnvVersion() != "000" {
		t.Error("EnvVersion not set")
	}
	if pack.Message.GetHostname() != "MyHostname" {
		t.Error("Hostname not set")
	}
	if pack.Message.GetSeverity() != 4 {
		t.Error("Severity not set")
	}
	if pack.Message.GetPid() != 12345 {
		t.Error("Pid not set")
	}
	var f []*message.Field
	f = pack.Message.FindAllFields("String")
	if len(f) != 1 || len(f[0].GetValueString()) != 1 || f[0].GetValueString()[0] != "foo" {
		t.Errorf("String field not set")
	}
	f = pack.Message.FindAllFields("Float")
	if len(f) != 1 || len(f[0].GetValueDouble()) != 1 || f[0].GetValueDouble()[0] != 1.2345 {
		t.Error("Float field not set")
	}
	f = pack.Message.FindAllFields("Int")
	if len(f) != 1 {
		t.Error("Int field not set")
	} else {
		if len(f[0].GetValueDouble()) != 2 || f[0].GetValueDouble()[0] != 123 ||
			f[0].GetValueDouble()[1] != 456 {
			t.Error("Int field set incorrectly")
		}
		if f[0].GetRepresentation() != "count" {
			t.Error("Int field representation not set")
		}
	}
	f = pack.Message.FindAllFields("Bool")
	if len(f) != 2 {
		t.Error("Bool fields not set")
	} else {
		if len(f[0].GetValueBool()) != 1 || !f[0].GetValueBool()[0] {
			t.Error("Bool field 0 not set")
		}
		if len(f[1].GetValueBool()) != 1 || f[1].GetValueBool()[0] {
			t.Error("Bool field 1 not set")
		}
	}
	if f = pack.Message.FindAllFields(""); len(f) != 1 {
		t.Error("No-name field not set")
	} else {
		if len(f[0].GetValueString()) != 1 || f[0].GetValueString()[0] != "bad idea" {
			t.Error("No-name field set incorrectly")
		}
	}
	if pack.Message.GetUuidString() != "550d19b9-58c7-49d8-b0dd-b48cd1c5b305" {
		t.Errorf("Uuid not set: %s", pack.Message.GetUuidString())
	}
}
Esempio n. 13
0
func TestWriteMessageErrors(t *testing.T) {
	pack := getTestPack()
	// NewPipelineConfig sets up Globals for error logging
	pipeline.NewPipelineConfig(nil)

	tests := []string{
		"too few parameters",
		"too many parameters",
		"Unknown field name",
		"Missing fields specifier",
		"Missing closing bracket",
		"Out of range field index",
		"Negative field index",
		"Negative array index",
		"nil field",
		"nil value",
		"empty uuid",
		"invalid uuid",
		"empty timestamp",
		"invalid timestamp",
		"bool severity",
		"double hostname",
	}
	msgs := []string{
		"process_message() ./testsupport/write_message_errors.lua:11: write_message() incorrect number of arguments",
		"process_message() ./testsupport/write_message_errors.lua:13: write_message() incorrect number of arguments",
		"process_message() ./testsupport/write_message_errors.lua:15: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:17: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:19: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:21: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:23: bad argument #4 to 'write_message' (field index must be >= 0)",
		"process_message() ./testsupport/write_message_errors.lua:25: bad argument #5 to 'write_message' (array index must be >= 0)",
		"process_message() ./testsupport/write_message_errors.lua:27: bad argument #1 to 'write_message' (string expected, got nil)",
		"process_message() ./testsupport/write_message_errors.lua:29: write_message() only accepts numeric, string, or boolean field values",
		"process_message() ./testsupport/write_message_errors.lua:31: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:33: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:35: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:37: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:39: write_message() failed",
		"process_message() ./testsupport/write_message_errors.lua:41: write_message() failed",
	}

	var sbc SandboxConfig
	sbc.ScriptFilename = "./testsupport/write_message_errors.lua"
	sbc.MemoryLimit = 32767
	sbc.InstructionLimit = 1000
	sbc.OutputLimit = 128
	for i, v := range tests {
		sb, err := lua.CreateLuaSandbox(&sbc)
		if err != nil {
			t.Errorf("%s", err)
		}
		err = sb.Init("", "decoder")
		if err != nil {
			t.Errorf("%s", err)
		}
		pack.Message.SetPayload(v)
		r := sb.ProcessMessage(pack)
		if r != 1 || STATUS_TERMINATED != sb.Status() {
			t.Errorf("test: %s status should be %d, received %d",
				v, STATUS_TERMINATED, sb.Status())
		}
		s := sb.LastError()
		if s != msgs[i] {
			t.Errorf("test: %s error should be \"%s\", received \"%s\"",
				v, msgs[i], s)
		}
		sb.Destroy("")
	}
}
Esempio n. 14
0
func NsqOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)
	var wg sync.WaitGroup

	errChan := make(chan error, 1)
	defer close(errChan)

	c.Specify("A nsq output", func() {
		output := new(NsqOutput)
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		config := output.ConfigStruct().(*NsqOutputConfig)
		config.Topic = "test"

		startOutput := func() {
			wg.Add(1)
			go func() {
				errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
		}

		var mockProducer *MockProducer
		output.newProducer = func(addr string, config *nsq.Config) (Producer, error) {
			mockProducer, _ = NewMockProducer(addr, config)
			return mockProducer, nil
		}

		msg := pipeline_ts.GetTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg

		c.Specify("requires at least one address", func() {
			err := output.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		config.Addresses = []string{"test.com:4150"}

		c.Specify("requires an encoder", func() {
			err := output.Init(config)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encoder().Return(nil)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("that is started", func() {
			encoder := new(plugins.PayloadEncoder)
			encoder.Init(&plugins.PayloadEncoderConfig{PrefixTs: false})
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))

			inChan := make(chan *pipeline.PipelinePack, 1)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)

			c.Specify("publishes a message with the configured topic", func() {
				err := output.Init(config)
				c.Assume(err, gs.IsNil)

				startOutput()

				inChan <- pack
				transaction := <-mockProducer.respChan
				c.Expect(transaction.Error, gs.IsNil)

				close(inChan)
				wg.Wait()
				c.Expect(<-errChan, gs.IsNil)
			})
		})

	})
}
Esempio n. 15
0
func OutputSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	pConfig := pipeline.NewPipelineConfig(nil)
	inChan := make(chan *pipeline.PipelinePack, 1)
	timer := make(chan time.Time, 1)

	c.Specify("A SandboxOutput", func() {
		output := new(SandboxOutput)
		output.SetPipelineConfig(pConfig)
		conf := output.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/testsupport/output.lua"
		conf.ModuleDirectory = "../lua/modules;../lua/testsupport/modules"
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		data := "1376389920 debug id=2321 url=example.com item=1"
		pack.Message.SetPayload(data)

		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
		oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
		oth.MockOutputRunner.EXPECT().Ticker().Return(timer)

		c.Specify("writes a payload to file", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Assume(err, gs.IsNil)

			tmpFile, err := os.Open("output.lua.txt")
			defer tmpFile.Close()
			c.Assume(err, gs.IsNil)
			contents, err := ioutil.ReadAll(tmpFile)
			c.Assume(err, gs.IsNil)
			c.Expect(string(contents), gs.Equals, data)
			c.Expect(output.processMessageCount, gs.Equals, int64(1))
			c.Expect(output.processMessageSamples, gs.Equals, int64(1))
			c.Expect(output.processMessageFailures, gs.Equals, int64(0))
		})

		c.Specify("failure processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			pack.Message.SetPayload("FAILURE")
			pack.BufferedPack = true
			pack.DelivErrChan = make(chan error, 1)
			inChan <- pack
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Assume(err, gs.IsNil)
			c.Expect(output.processMessageFailures, gs.Equals, int64(1))
			err = <-pack.DelivErrChan
			c.Expect(err.Error(), gs.Equals, "failure message")
		})

		c.Specify("user abort processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			e := errors.New("FATAL: user abort")
			pack.Message.SetPayload("USERABORT")
			inChan <- pack
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err.Error(), gs.Equals, e.Error())
		})

		c.Specify("fatal error processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			pack.Message.SetPayload("FATAL")
			inChan <- pack
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("fatal error in timer_event", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			timer <- time.Now()
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("fatal error in shutdown timer_event", func() {
			conf.TimerEventOnShutdown = true
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}
Esempio n. 16
0
File: main.go Progetto: KushalP/heka
func main() {
	configPath := flag.String("config", filepath.FromSlash("/etc/hekad.toml"),
		"Config file or directory. If directory is specified then all files "+
			"in the directory will be loaded.")
	version := flag.Bool("version", false, "Output version and exit")
	flag.Parse()

	config := &HekadConfig{}
	var err error
	var cpuProfName string
	var memProfName string

	if flag.NFlag() == 0 {
		flag.PrintDefaults()
		os.Exit(0)
	}

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	config, err = LoadHekadConfig(*configPath)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	globals, cpuProfName, memProfName := setGlobalConfigs(config)

	if err = os.MkdirAll(globals.BaseDir, 0755); err != nil {
		log.Fatalf("Error creating base_dir %s: %s", config.BaseDir, err)
	}

	if cpuProfName != "" {
		profFile, err := os.Create(cpuProfName)
		if err != nil {
			log.Fatalln(err)
		}
		profFile.Close()
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if memProfName != "" {
		defer func() {
			profFile, err := os.Create(memProfName)
			if err != nil {
				log.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Set up and load the pipeline configuration and start the daemon.
	pipeconf := pipeline.NewPipelineConfig(globals)
	p, err := os.Open(*configPath)
	fi, err := p.Stat()

	if fi.IsDir() {
		files, _ := ioutil.ReadDir(*configPath)
		for _, f := range files {
			err = pipeconf.LoadFromConfigFile(filepath.Join(*configPath, f.Name()))
		}
	} else {
		err = pipeconf.LoadFromConfigFile(*configPath)
	}

	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}
Esempio n. 17
0
func DecoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxDecoder", func() {

		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)

		c.Specify("that uses lpeg and inject_message", func() {
			dRunner.EXPECT().Name().Return("serialize")
			conf.ScriptFilename = "../lua/testsupport/decoder.lua"
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			c.Specify("decodes simple messages", func() {
				data := "1376389920 debug id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				_, err = decoder.Decode(pack)
				c.Assume(err, gs.IsNil)

				c.Expect(pack.Message.GetTimestamp(),
					gs.Equals,
					int64(1376389920000000000))

				c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

				var ok bool
				var value interface{}
				value, ok = pack.Message.GetFieldValue("id")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "2321")

				value, ok = pack.Message.GetFieldValue("url")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "example.com")

				value, ok = pack.Message.GetFieldValue("item")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "1")
				decoder.Shutdown()
			})

			c.Specify("decodes an invalid messages", func() {
				data := "1376389920 bogus id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(len(packs), gs.Equals, 0)
				c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
				c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
				decoder.Shutdown()
			})

			c.Specify("Preserves data", func() {
				conf.ScriptFilename = "../lua/testsupport/serialize.lua"
				conf.PreserveData = true
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)
				decoder.Shutdown()
				_, err = os.Stat("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
				err = os.Remove("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("that only uses write_message", func() {
			conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
			dRunner.EXPECT().Name().Return("write_message")
			err := decoder.Init(conf)
			decoder.SetDecoderRunner(dRunner)
			c.Assume(err, gs.IsNil)

			c.Specify("adds a string field to the message", func() {
				data := "string field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(string), gs.Equals, "foo")
			})

			c.Specify("adds a numeric field to the message", func() {
				data := "num field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(float64), gs.Equals, float64(1))
			})

			c.Specify("adds a boolean field to the message", func() {
				data := "bool field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(bool), gs.Equals, true)
			})

			c.Specify("sets type and payload", func() {
				data := "set type and payload"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
				c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
			})

			c.Specify("sets field value with representation", func() {
				data := "set field value with representation"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("rep")
				c.Expect(len(fields), gs.Equals, 1)
				field := fields[0]
				values := field.GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "foo")
				c.Expect(field.GetRepresentation(), gs.Equals, "representation")
			})

			c.Specify("sets multiple field string values", func() {
				data := "set multiple field string values"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("multi")
				c.Expect(len(fields), gs.Equals, 2)
				values := fields[0].GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "first")
				values = fields[1].GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "second")
			})

			c.Specify("sets field string array value", func() {
				data := "set field string array value"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("array")
				c.Expect(len(fields), gs.Equals, 1)
				values := fields[0].GetValueString()
				c.Expect(len(values), gs.Equals, 2)
				c.Expect(values[0], gs.Equals, "first")
				c.Expect(values[1], gs.Equals, "second")
			})
		})
	})

	c.Specify("A Multipack SandboxDecoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
		supply := make(chan *pipeline.PipelinePack, 3)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message = getTestMessage()

		pack1 := pipeline.NewPipelinePack(supply)
		pack2 := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")

		c.Specify("decodes into multiple packs", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			decoder.SetDecoderRunner(dRunner)
			gomock.InOrder(
				dRunner.EXPECT().NewPack().Return(pack1),
				dRunner.EXPECT().NewPack().Return(pack2),
			)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 3)
			c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
			c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
			c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")

			for i := 0; i < 1; i++ {
				c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
				c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
				c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
				c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))

			}
			decoder.Shutdown()
		})
	})

	c.Specify("Linux Cpu Stats decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/linux_loadavg.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes a message", func() {
			payload := "0.00 0.01 0.05 3/153 660\n"
			pack.Message.SetPayload(payload)
			f, err := message.NewField("FilePath", "/proc/loadavg", "")
			c.Assume(err, gs.IsNil)
			pack.Message.AddField(f)

			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)
			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("1MinAvg")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 0.00)

			value, ok = pack.Message.GetFieldValue("5MinAvg")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 0.01)

			value, ok = pack.Message.GetFieldValue("15MinAvg")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 0.05)

			value, ok = pack.Message.GetFieldValue("NumProcesses")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(3))

			value, ok = pack.Message.GetFieldValue("FilePath")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, "/proc/loadavg")
		})

		c.Specify("decodes an invalid message", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Linux Mem Stats decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/linux_memstats.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes a message", func() {
			payload := `MemTotal:        4047616 kB
MemFree:         3135780 kB
HugePages_Free:        0
`
			pack.Message.SetPayload(payload)
			f, err := message.NewField("FilePath", "/proc/meminfo", "")
			c.Assume(err, gs.IsNil)
			pack.Message.AddField(f)

			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)
			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("MemTotal")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 4.047616e+06)

			value, ok = pack.Message.GetFieldValue("MemFree")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, 3.13578e+06)

			value, ok = pack.Message.GetFieldValue("HugePages_Free")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("FilePath")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, "/proc/meminfo")
		})

		c.Specify("decodes an invalid message", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Linux Disk Stats decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/linux_diskstats.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes a message", func() {
			payload := "   13903    11393   969224    49444    10780    10161  1511920     4104        0     5064    53468\n"
			pack.Message.SetPayload(payload)
			f, err := message.NewField("FilePath", "/sys/block/sda/stat", "")
			c.Assume(err, gs.IsNil)
			pack.Message.AddField(f)

			f, err = message.NewField("TickerInterval", int64(2), "")
			c.Assume(err, gs.IsNil)
			pack.Message.AddField(f)

			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)
			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			// These are in the same order the payload should be
			value, ok = pack.Message.GetFieldValue("ReadsCompleted")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(13903))

			value, ok = pack.Message.GetFieldValue("ReadsMerged")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(11393))

			value, ok = pack.Message.GetFieldValue("SectorsRead")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(969224))

			value, ok = pack.Message.GetFieldValue("TimeReading")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(49444))

			value, ok = pack.Message.GetFieldValue("WritesCompleted")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(10780))

			value, ok = pack.Message.GetFieldValue("WritesMerged")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(10161))

			value, ok = pack.Message.GetFieldValue("SectorsWritten")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(1511920))

			value, ok = pack.Message.GetFieldValue("TimeWriting")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(4104))

			value, ok = pack.Message.GetFieldValue("NumIOInProgress")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("TimeDoingIO")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(5064))

			value, ok = pack.Message.GetFieldValue("WeightedTimeDoingIO")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(53468))

			value, ok = pack.Message.GetFieldValue("TickerInterval")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(2))

			value, ok = pack.Message.GetFieldValue("FilePath")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, "/sys/block/sda/stat")
		})

		c.Specify("decodes a message with no leading space", func() {
			payload := "19092852        0 510563170 15817012 46452019        0 1546950712 262535124        0 23823976 278362684\n"
			pack.Message.SetPayload(payload)

			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			value, ok := pack.Message.GetFieldValue("ReadsCompleted")
			c.Expect(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, float64(19092852))
		})

		c.Specify("decodes an invalid message", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Nginx access log decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
		conf.Config["user_agent_transform"] = true
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392050801000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("remote_addr")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "127.0.0.1")

			value, ok = pack.Message.GetFieldValue("user_agent_browser")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Firefox")
			value, ok = pack.Message.GetFieldValue("user_agent_version")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(26))
			value, ok = pack.Message.GetFieldValue("user_agent_os")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Linux")
			_, ok = pack.Message.GetFieldValue("http_user_agent")
			c.Expect(ok, gs.Equals, false)

			value, ok = pack.Message.GetFieldValue("body_bytes_sent")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("status")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(304))
			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Apache access log decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/apache_access.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
		conf.Config["user_agent_transform"] = true
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392050801000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("remote_addr")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "127.0.0.1")

			value, ok = pack.Message.GetFieldValue("user_agent_browser")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Firefox")
			value, ok = pack.Message.GetFieldValue("user_agent_version")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(26))
			value, ok = pack.Message.GetFieldValue("user_agent_os")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Linux")
			_, ok = pack.Message.GetFieldValue("http_user_agent")
			c.Expect(ok, gs.Equals, false)

			value, ok = pack.Message.GetFieldValue("body_bytes_sent")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("status")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(304))
			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("rsyslog decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["type"] = "MyTestFormat"
		conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
		conf.Config["tz"] = "America/Los_Angeles"
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n"
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			// Syslog timestamp doesn't support year, so we have to calculate
			// it for the current year or else this test will fail every
			// January.
			year := time.Now().Year()
			tStr := fmt.Sprintf("%d Feb 10 12:58:58 -0800", year)
			t, err := time.Parse("2006 Jan 02 15:04:05 -0700", tStr)
			c.Assume(err, gs.IsNil)
			unixT := t.UnixNano()

			c.Expect(pack.Message.GetTimestamp(), gs.Equals, unixT)

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
			c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
			c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.")
			c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"])

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("programname")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "widget")

			value, ok = pack.Message.GetFieldValue("syslogfacility")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(3))

			value, ok = pack.Message.GetFieldValue("timegenerated")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(1392065939000000000))

			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("mysql decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["truncate_sql"] = int64(5)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decode standard slow query log", func() {
			data := `# User@Host: syncrw[syncrw] @  [127.0.0.1]
# Query_time: 2.964652  Lock_time: 0.000050 Rows_sent: 251  Rows_examined: 9773
use widget;
SET last_insert_id=999,insert_id=1000,timestamp=1399500744;
# administrator command: do something
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;`
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1399500744000000000))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
			c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query")

			decoder.Shutdown()
		})
	})

	c.Specify("mariadb decoder", func() {
		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["truncate_sql"] = int64(5)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decode standard slow query log", func() {
			data := `# User@Host: syncrw[syncrw] @  [127.0.0.1]
# Thread_id: 110804  Schema: weave0  QC_hit: No
# Query_time: 1.178108  Lock_time: 0.000053  Rows_sent: 198  Rows_examined: 198
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;`
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1399500744000000000))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
			c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query")

			decoder.Shutdown()
		})
	})
}
Esempio n. 18
0
func EncoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// NewPipelineConfig sets up Globals which is needed for the
	// pipeline.Prepend*Dir functions to not die during plugin Init().
	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxEncoder", func() {

		encoder := new(SandboxEncoder)
		encoder.SetPipelineConfig(pConfig)
		conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message.SetPayload("original")
		pack.Message.SetType("my_type")
		pack.Message.SetPid(12345)
		pack.Message.SetSeverity(4)
		pack.Message.SetHostname("hostname")
		pack.Message.SetTimestamp(54321)
		pack.Message.SetUuid(uuid.NewRandom())
		var (
			result []byte
			err    error
		)

		c.Specify("emits JSON correctly", func() {
			conf.ScriptFilename = "../lua/testsupport/encoder_json.lua"
			err = encoder.Init(conf)
			c.Expect(err, gs.IsNil)

			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			msg := new(message.Message)
			err = json.Unmarshal(result, msg)
			c.Expect(err, gs.IsNil)
			c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
			c.Expect(msg.GetPid(), gs.Equals, int32(12345))
			c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
			c.Expect(msg.GetHostname(), gs.Equals, "hostname")
			c.Expect(msg.GetPayload(), gs.Equals, "original")
			c.Expect(msg.GetType(), gs.Equals, "my_type")
		})

		c.Specify("emits text correctly", func() {
			conf.ScriptFilename = "../lua/testsupport/encoder_text.lua"
			err = encoder.Init(conf)
			c.Expect(err, gs.IsNil)

			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(string(result), gs.Equals, "Prefixed original")
		})

		c.Specify("emits protobuf correctly", func() {

			c.Specify("when inject_message is used", func() {
				conf.ScriptFilename = "../lua/testsupport/encoder_protobuf.lua"
				err = encoder.Init(conf)
				c.Expect(err, gs.IsNil)

				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				msg := new(message.Message)
				err = proto.Unmarshal(result, msg)
				c.Expect(err, gs.IsNil)
				c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
				c.Expect(msg.GetPid(), gs.Equals, int32(12345))
				c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
				c.Expect(msg.GetHostname(), gs.Equals, "hostname")
				c.Expect(msg.GetPayload(), gs.Equals, "mutated")
				c.Expect(msg.GetType(), gs.Equals, "after")
			})

			c.Specify("when `write_message` is used", func() {
				conf.ScriptFilename = "../lua/testsupport/encoder_writemessage.lua"
				err = encoder.Init(conf)
				c.Expect(err, gs.IsNil)

				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				msg := new(message.Message)
				err = proto.Unmarshal(result, msg)
				c.Expect(err, gs.IsNil)
				c.Expect(msg.GetPayload(), gs.Equals, "mutated payload")
				c.Expect(pack.Message.GetPayload(), gs.Equals, "original")
			})
		})
	})

	c.Specify("cbuf librato encoder", func() {
		encoder := new(SandboxEncoder)
		encoder.SetPipelineConfig(pConfig)
		conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message.SetType("my_type")
		pack.Message.SetPid(12345)
		pack.Message.SetSeverity(4)
		pack.Message.SetHostname("hostname")
		pack.Message.SetTimestamp(54321)
		pack.Message.SetUuid(uuid.NewRandom())
		var (
			result []byte
			err    error
		)
		conf.ScriptFilename = "../lua/encoders/cbuf_librato.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.Config = make(map[string]interface{})
		err = encoder.Init(conf)
		c.Assume(err, gs.IsNil)

		c.Specify("encodes cbuf data", func() {
			payload := `{"time":1410823460,"rows":5,"columns":5,"seconds_per_row":5,"column_info":[{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
1	2	3	4	5
6	7	8	9	10
11	12	13	14	15
16	17	18	19	20
21	22	23	24	25
`
			pack.Message.SetPayload(payload)
			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			expected := `{"gauges":[{"value":1,"measure_time":1410823460,"name":"HTTP_200","source":"hostname"},{"value":2,"measure_time":1410823460,"name":"HTTP_300","source":"hostname"},{"value":3,"measure_time":1410823460,"name":"HTTP_400","source":"hostname"},{"value":4,"measure_time":1410823460,"name":"HTTP_500","source":"hostname"},{"value":5,"measure_time":1410823460,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":6,"measure_time":1410823465,"name":"HTTP_200","source":"hostname"},{"value":7,"measure_time":1410823465,"name":"HTTP_300","source":"hostname"},{"value":8,"measure_time":1410823465,"name":"HTTP_400","source":"hostname"},{"value":9,"measure_time":1410823465,"name":"HTTP_500","source":"hostname"},{"value":10,"measure_time":1410823465,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":11,"measure_time":1410823470,"name":"HTTP_200","source":"hostname"},{"value":12,"measure_time":1410823470,"name":"HTTP_300","source":"hostname"},{"value":13,"measure_time":1410823470,"name":"HTTP_400","source":"hostname"},{"value":14,"measure_time":1410823470,"name":"HTTP_500","source":"hostname"},{"value":15,"measure_time":1410823470,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":16,"measure_time":1410823475,"name":"HTTP_200","source":"hostname"},{"value":17,"measure_time":1410823475,"name":"HTTP_300","source":"hostname"},{"value":18,"measure_time":1410823475,"name":"HTTP_400","source":"hostname"},{"value":19,"measure_time":1410823475,"name":"HTTP_500","source":"hostname"},{"value":20,"measure_time":1410823475,"name":"HTTP_UNKNOWN","source":"hostname"}]}`
			c.Expect(string(result), gs.Equals, expected)

			c.Specify("and correctly advances", func() {
				payload := `{"time":1410823475,"rows":5,"columns":5,"seconds_per_row":5,"column_info":[{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_300","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"},{"name":"HTTP_UNKNOWN","unit":"count","aggregation":"sum"}]}
16	17	18	19	20
21	22	23	24	25
1	2	3	4	5
6	nan	8	nan	10
5	4	3	2	1
`
				pack.Message.SetPayload(payload)
				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)
				expected := `{"gauges":[{"value":21,"measure_time":1410823480,"name":"HTTP_200","source":"hostname"},{"value":22,"measure_time":1410823480,"name":"HTTP_300","source":"hostname"},{"value":23,"measure_time":1410823480,"name":"HTTP_400","source":"hostname"},{"value":24,"measure_time":1410823480,"name":"HTTP_500","source":"hostname"},{"value":25,"measure_time":1410823480,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":1,"measure_time":1410823485,"name":"HTTP_200","source":"hostname"},{"value":2,"measure_time":1410823485,"name":"HTTP_300","source":"hostname"},{"value":3,"measure_time":1410823485,"name":"HTTP_400","source":"hostname"},{"value":4,"measure_time":1410823485,"name":"HTTP_500","source":"hostname"},{"value":5,"measure_time":1410823485,"name":"HTTP_UNKNOWN","source":"hostname"},{"value":6,"measure_time":1410823490,"name":"HTTP_200","source":"hostname"},{"value":8,"measure_time":1410823490,"name":"HTTP_400","source":"hostname"},{"value":10,"measure_time":1410823490,"name":"HTTP_UNKNOWN","source":"hostname"}]}`
				c.Expect(string(result), gs.Equals, expected)
			})
		})
	})

	c.Specify("schema influx encoder", func() {
		encoder := new(SandboxEncoder)
		encoder.SetPipelineConfig(pConfig)
		conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message.SetType("my_type")
		pack.Message.SetPid(12345)
		pack.Message.SetSeverity(4)
		pack.Message.SetHostname("hostname")
		pack.Message.SetTimestamp(54321 * 1e9)
		pack.Message.SetLogger("Logger")
		pack.Message.SetPayload("Payload value lorem ipsum")

		f, err := message.NewField("intField", 123, "")
		c.Assume(err, gs.IsNil)
		err = f.AddValue(456)
		c.Assume(err, gs.IsNil)
		pack.Message.AddField(f)

		f, err = message.NewField("strField", "0_first", "")
		c.Assume(err, gs.IsNil)
		err = f.AddValue("0_second")
		c.Assume(err, gs.IsNil)
		pack.Message.AddField(f)

		f, err = message.NewField("strField", "1_first", "")
		c.Assume(err, gs.IsNil)
		err = f.AddValue("1_second")
		c.Assume(err, gs.IsNil)
		pack.Message.AddField(f)

		f, err = message.NewField("byteField", []byte("first"), "")
		c.Assume(err, gs.IsNil)
		err = f.AddValue([]byte("second"))
		c.Assume(err, gs.IsNil)
		pack.Message.AddField(f)

		conf.ScriptFilename = "../lua/encoders/schema_influx.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.Config = make(map[string]interface{})

		c.Specify("encodes a basic message", func() {
			err = encoder.Init(conf)
			c.Assume(err, gs.IsNil)
			result, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			expected := `[{"points":[[54321000,"my_type","Payload value lorem ipsum","hostname",12345,"Logger",4,"",[123,456],["0_first","0_second"],["1_first","1_second"]]],"name":"series","columns":["time","Type","Payload","Hostname","Pid","Logger","Severity","EnvVersion","intField","strField","strField2"]}]`
			c.Expect(string(result), gs.Equals, expected)
		})

		c.Specify("interpolates series name correctly", func() {
			conf.Config["series"] = "series.%{Pid}.%{Type}.%{strField}.%{intField}"
			err = encoder.Init(conf)
			c.Assume(err, gs.IsNil)
			result, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			expected := `[{"points":[[54321000,"my_type","Payload value lorem ipsum","hostname",12345,"Logger",4,"",[123,456],["0_first","0_second"],["1_first","1_second"]]],"name":"series.12345.my_type.0_first.123","columns":["time","Type","Payload","Hostname","Pid","Logger","Severity","EnvVersion","intField","strField","strField2"]}]`
			c.Expect(string(result), gs.Equals, expected)
		})

		c.Specify("skips specified correctly", func() {
			conf.Config["skip_fields"] = "Payload strField Type"
			err = encoder.Init(conf)
			c.Assume(err, gs.IsNil)
			result, err := encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			expected := `[{"points":[[54321000,"hostname",12345,"Logger",4,"",[123,456]]],"name":"series","columns":["time","Hostname","Pid","Logger","Severity","EnvVersion","intField"]}]`
			c.Expect(string(result), gs.Equals, expected)
		})
	})
}
Esempio n. 19
0
func main() {
	configPath := flag.String("config", filepath.FromSlash("/etc/hekad.toml"),
		"Config file or directory. If directory is specified then all files "+
			"in the directory will be loaded.")
	version := flag.Bool("version", false, "Output version and exit")
	flag.Parse()

	config := &HekadConfig{}
	var err error
	var cpuProfName string
	var memProfName string

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	config, err = LoadHekadConfig(*configPath)
	if err != nil {
		pipeline.LogError.Fatal("Error reading config: ", err)
	}
	if config.SampleDenominator <= 0 {
		pipeline.LogError.Fatalln("'sample_denominator' value must be greater than 0.")
	}
	globals, cpuProfName, memProfName := setGlobalConfigs(config)

	if err = os.MkdirAll(globals.BaseDir, 0755); err != nil {
		pipeline.LogError.Fatalf("Error creating 'base_dir' %s: %s", config.BaseDir, err)
	}

	if config.MaxMessageSize > 1024 {
		message.SetMaxMessageSize(config.MaxMessageSize)
	} else if config.MaxMessageSize > 0 {
		pipeline.LogError.Fatalln("Error: 'max_message_size' setting must be greater than 1024.")
	}
	if config.PidFile != "" {
		contents, err := ioutil.ReadFile(config.PidFile)
		if err == nil {
			pid, err := strconv.Atoi(strings.TrimSpace(string(contents)))
			if err != nil {
				pipeline.LogError.Fatalf("Error reading proccess id from pidfile '%s': %s",
					config.PidFile, err)
			}

			process, err := os.FindProcess(pid)

			// on Windows, err != nil if the process cannot be found
			if runtime.GOOS == "windows" {
				if err == nil {
					pipeline.LogError.Fatalf("Process %d is already running.", pid)
				}
			} else if process != nil {
				// err is always nil on POSIX, so we have to send the process
				// a signal to check whether it exists
				if err = process.Signal(syscall.Signal(0)); err == nil {
					pipeline.LogError.Fatalf("Process %d is already running.", pid)
				}
			}
		}
		if err = ioutil.WriteFile(config.PidFile, []byte(strconv.Itoa(os.Getpid())),
			0644); err != nil {

			pipeline.LogError.Fatalf("Unable to write pidfile '%s': %s", config.PidFile, err)
		}
		pipeline.LogInfo.Printf("Wrote pid to pidfile '%s'", config.PidFile)
		defer func() {
			if err = os.Remove(config.PidFile); err != nil {
				pipeline.LogError.Printf("Unable to remove pidfile '%s': %s", config.PidFile, err)
			}
		}()
	}

	if cpuProfName != "" {
		profFile, err := os.Create(cpuProfName)
		if err != nil {
			pipeline.LogError.Fatalln(err)
		}

		pprof.StartCPUProfile(profFile)
		defer func() {
			pprof.StopCPUProfile()
			profFile.Close()
		}()
	}

	if memProfName != "" {
		defer func() {
			profFile, err := os.Create(memProfName)
			if err != nil {
				pipeline.LogError.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Set up and load the pipeline configuration and start the daemon.
	pipeconf := pipeline.NewPipelineConfig(globals)
	if err = loadFullConfig(pipeconf, configPath); err != nil {
		pipeline.LogError.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}
Esempio n. 20
0
func DecoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// NewPipelineConfig sets up Globals which is needed for the
	// pipeline.Prepend*Dir functions to not die during plugin Init().
	_ = pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxDecoder", func() {

		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)

		c.Specify("that uses lpeg and inject_message", func() {
			dRunner.EXPECT().Name().Return("serialize")
			conf.ScriptFilename = "../lua/testsupport/decoder.lua"
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			c.Specify("decodes simple messages", func() {
				data := "1376389920 debug id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				_, err = decoder.Decode(pack)
				c.Assume(err, gs.IsNil)

				c.Expect(pack.Message.GetTimestamp(),
					gs.Equals,
					int64(1376389920000000000))

				c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

				var ok bool
				var value interface{}
				value, ok = pack.Message.GetFieldValue("id")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "2321")

				value, ok = pack.Message.GetFieldValue("url")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "example.com")

				value, ok = pack.Message.GetFieldValue("item")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "1")
				decoder.Shutdown()
			})

			c.Specify("decodes an invalid messages", func() {
				data := "1376389920 bogus id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(len(packs), gs.Equals, 0)
				c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
				c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
				decoder.Shutdown()
			})

			c.Specify("Preserves data", func() {
				conf.ScriptFilename = "../lua/testsupport/serialize.lua"
				conf.PreserveData = true
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)
				decoder.Shutdown()
				_, err = os.Stat("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
				err = os.Remove("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("that only uses write_message", func() {
			conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
			dRunner.EXPECT().Name().Return("write_message")
			err := decoder.Init(conf)
			decoder.SetDecoderRunner(dRunner)
			c.Assume(err, gs.IsNil)

			c.Specify("adds a string field to the message", func() {
				data := "string field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(string), gs.Equals, "foo")
			})

			c.Specify("adds a numeric field to the message", func() {
				data := "num field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(float64), gs.Equals, float64(1))
			})

			c.Specify("adds a boolean field to the message", func() {
				data := "bool field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(bool), gs.Equals, true)
			})

			c.Specify("sets type and payload", func() {
				data := "set type and payload"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				c.Expect(pack.Message.GetType(), gs.Equals, "my_type")
				c.Expect(pack.Message.GetPayload(), gs.Equals, "my_payload")
			})

			c.Specify("sets field value with representation", func() {
				data := "set field value with representation"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("rep")
				c.Expect(len(fields), gs.Equals, 1)
				field := fields[0]
				values := field.GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "foo")
				c.Expect(field.GetRepresentation(), gs.Equals, "representation")
			})

			c.Specify("sets multiple field string values", func() {
				data := "set multiple field string values"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("multi")
				c.Expect(len(fields), gs.Equals, 2)
				values := fields[0].GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "first")
				values = fields[1].GetValueString()
				c.Expect(len(values), gs.Equals, 1)
				c.Expect(values[0], gs.Equals, "second")
			})

			c.Specify("sets field string array value", func() {
				data := "set field string array value"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				fields := pack.Message.FindAllFields("array")
				c.Expect(len(fields), gs.Equals, 1)
				values := fields[0].GetValueString()
				c.Expect(len(values), gs.Equals, 2)
				c.Expect(values[0], gs.Equals, "first")
				c.Expect(values[1], gs.Equals, "second")
			})
		})
	})

	c.Specify("A Multipack SandboxDecoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/testsupport/multipack_decoder.lua"
		supply := make(chan *pipeline.PipelinePack, 3)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message = getTestMessage()

		pack1 := pipeline.NewPipelinePack(supply)
		pack2 := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")

		c.Specify("decodes into multiple packs", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			decoder.SetDecoderRunner(dRunner)
			gomock.InOrder(
				dRunner.EXPECT().NewPack().Return(pack1),
				dRunner.EXPECT().NewPack().Return(pack2),
			)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 3)
			c.Expect(packs[0].Message.GetPayload(), gs.Equals, "message one")
			c.Expect(packs[1].Message.GetPayload(), gs.Equals, "message two")
			c.Expect(packs[2].Message.GetPayload(), gs.Equals, "message three")

			for i := 0; i < 1; i++ {
				c.Expect(packs[i].Message.GetType(), gs.Equals, "TEST")
				c.Expect(packs[i].Message.GetHostname(), gs.Equals, "my.host.name")
				c.Expect(packs[i].Message.GetLogger(), gs.Equals, "GoSpec")
				c.Expect(packs[i].Message.GetSeverity(), gs.Equals, int32(6))

			}
			decoder.Shutdown()
		})
	})

	c.Specify("Nginx access log decoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/nginx_access.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["log_format"] = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
		conf.Config["user_agent_transform"] = true
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392050801000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("remote_addr")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "127.0.0.1")

			value, ok = pack.Message.GetFieldValue("user_agent_browser")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Firefox")
			value, ok = pack.Message.GetFieldValue("user_agent_version")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(26))
			value, ok = pack.Message.GetFieldValue("user_agent_os")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Linux")
			_, ok = pack.Message.GetFieldValue("http_user_agent")
			c.Expect(ok, gs.Equals, false)

			value, ok = pack.Message.GetFieldValue("body_bytes_sent")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("status")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(304))
			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("Apache access log decoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/apache_access.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["log_format"] = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""
		conf.Config["user_agent_transform"] = true
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0\""
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392050801000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("remote_addr")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "127.0.0.1")

			value, ok = pack.Message.GetFieldValue("user_agent_browser")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Firefox")
			value, ok = pack.Message.GetFieldValue("user_agent_version")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(26))
			value, ok = pack.Message.GetFieldValue("user_agent_os")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "Linux")
			_, ok = pack.Message.GetFieldValue("http_user_agent")
			c.Expect(ok, gs.Equals, false)

			value, ok = pack.Message.GetFieldValue("body_bytes_sent")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(0))

			value, ok = pack.Message.GetFieldValue("status")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(304))
			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("rsyslog decoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/rsyslog.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["type"] = "MyTestFormat"
		conf.Config["template"] = "%pri% %TIMESTAMP% %TIMEGENERATED:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
		conf.Config["tz"] = "America/Los_Angeles"
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decodes simple messages", func() {
			data := "28 Feb 10 12:58:58 2014-02-10T12:58:59-08:00 testhost widget[4322]: test message.\n"
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1392065938000000000))

			c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(4))
			c.Expect(pack.Message.GetHostname(), gs.Equals, "testhost")
			c.Expect(pack.Message.GetPid(), gs.Equals, int32(4322))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "test message.")
			c.Expect(pack.Message.GetType(), gs.Equals, conf.Config["type"])

			var ok bool
			var value interface{}
			value, ok = pack.Message.GetFieldValue("programname")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, "widget")

			value, ok = pack.Message.GetFieldValue("syslogfacility")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(3))

			value, ok = pack.Message.GetFieldValue("timegenerated")
			c.Expect(ok, gs.Equals, true)
			c.Expect(value, gs.Equals, float64(1392065939000000000))

			decoder.Shutdown()
		})

		c.Specify("decodes an invalid messages", func() {
			data := "bogus message"
			pack.Message.SetPayload(data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
			c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
			decoder.Shutdown()
		})
	})

	c.Specify("mysql decoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/mysql_slow_query.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["truncate_sql"] = int64(5)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decode standard slow query log", func() {
			data := `# User@Host: syncrw[syncrw] @  [127.0.0.1]
# Query_time: 2.964652  Lock_time: 0.000050 Rows_sent: 251  Rows_examined: 9773
use widget;
SET last_insert_id=999,insert_id=1000,timestamp=1399500744;
# administrator command: do something
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;
`
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1399500744000000000))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
			c.Expect(pack.Message.GetType(), gs.Equals, "mysql.slow-query")

			decoder.Shutdown()
		})
	})

	c.Specify("mariadb decoder", func() {
		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/decoders/mariadb_slow_query.lua"
		conf.ModuleDirectory = "../../../../../../modules"
		conf.MemoryLimit = 8e6
		conf.Config = make(map[string]interface{})
		conf.Config["truncate_sql"] = int64(5)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		dRunner.EXPECT().Name().Return("SandboxDecoder")
		err := decoder.Init(conf)
		c.Assume(err, gs.IsNil)
		decoder.SetDecoderRunner(dRunner)

		c.Specify("decode standard slow query log", func() {
			data := `# User@Host: syncrw[syncrw] @  [127.0.0.1]
# Thread_id: 110804  Schema: weave0  QC_hit: No
# Query_time: 1.178108  Lock_time: 0.000053  Rows_sent: 198  Rows_examined: 198
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;
`
			pack.Message.SetPayload(data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			c.Expect(pack.Message.GetTimestamp(),
				gs.Equals,
				int64(1399500744000000000))
			c.Expect(pack.Message.GetPayload(), gs.Equals, "/* [q...")
			c.Expect(pack.Message.GetType(), gs.Equals, "mariadb.slow-query")

			decoder.Shutdown()
		})
	})
}
Esempio n. 21
0
func FilterSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	fth := NewFilterTestHelper(ctrl)
	inChan := make(chan *pipeline.PipelinePack, 1)
	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxFilter", func() {
		sbFilter := new(SandboxFilter)
		config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
		config.ScriptType = "lua"
		config.MemoryLimit = 32000
		config.InstructionLimit = 1000
		config.OutputLimit = 1024

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("Over inject messages from ProcessMessage", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(3)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(2)
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("exceeded InjectMessage count"))

			config.ScriptFilename = "../lua/testsupport/processinject.lua"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
		})

		c.Specify("Over inject messages from TimerEvent", func() {
			var timer <-chan time.Time
			timer = time.Tick(time.Duration(1) * time.Millisecond)
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(12)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack).Times(11)
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("exceeded InjectMessage count"))

			config.ScriptFilename = "../lua/testsupport/timerinject.lua"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			go func() {
				time.Sleep(time.Duration(250) * time.Millisecond)
				close(inChan)
			}()
			sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
		})

		c.Specify("Preserves data", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)

			config.ScriptFilename = "../lua/testsupport/serialize.lua"
			config.PreserveData = true
			sbFilter.SetName("serialize")
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			close(inChan)
			sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			_, err = os.Stat("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
			err = os.Remove("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
		})
	})

	c.Specify("A SandboxManagerFilter", func() {
		sbmFilter := new(SandboxManagerFilter)
		config := sbmFilter.ConfigStruct().(*SandboxManagerFilterConfig)
		config.MaxFilters = 1

		origBaseDir := pipeline.Globals().BaseDir
		pipeline.Globals().BaseDir = os.TempDir()
		sbxMgrsDir := filepath.Join(pipeline.Globals().BaseDir, "sbxmgrs")
		defer func() {
			pipeline.Globals().BaseDir = origBaseDir
			tmpErr := os.RemoveAll(sbxMgrsDir)
			c.Expect(tmpErr, gs.IsNil)
		}()

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg
		pack.Decoded = true

		c.Specify("Control message in the past", func() {
			sbmFilter.Init(config)
			pack.Message.SetTimestamp(time.Now().UnixNano() - 5e9)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: 5 seconds skew"))
			inChan <- pack
			close(inChan)
			sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
		})

		c.Specify("Control message in the future", func() {
			sbmFilter.Init(config)
			pack.Message.SetTimestamp(time.Now().UnixNano() + 5.9e9)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().Name().Return("SandboxManagerFilter")
			fth.MockFilterRunner.EXPECT().LogError(fmt.Errorf("Discarded control message: -5 seconds skew"))
			inChan <- pack
			close(inChan)
			sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
		})

		c.Specify("Generates the right default working directory", func() {
			sbmFilter.Init(config)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			name := "SandboxManagerFilter"
			fth.MockFilterRunner.EXPECT().Name().Return(name)
			close(inChan)
			sbmFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(sbmFilter.workingDirectory, gs.Equals, sbxMgrsDir)
			_, err := os.Stat(sbxMgrsDir)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Sanity check the default sandbox configuration limits", func() {
			sbmFilter.Init(config)
			c.Expect(sbmFilter.memoryLimit, gs.Equals, uint(8*1024*1024))
			c.Expect(sbmFilter.instructionLimit, gs.Equals, uint(1e6))
			c.Expect(sbmFilter.outputLimit, gs.Equals, uint(63*1024))
		})

		c.Specify("Sanity check the user specified sandbox configuration limits", func() {
			config.MemoryLimit = 123456
			config.InstructionLimit = 4321
			config.OutputLimit = 8765
			sbmFilter.Init(config)
			c.Expect(sbmFilter.memoryLimit, gs.Equals, config.MemoryLimit)
			c.Expect(sbmFilter.instructionLimit, gs.Equals, config.InstructionLimit)
			c.Expect(sbmFilter.outputLimit, gs.Equals, config.OutputLimit)
		})
	})
}
Esempio n. 22
0
func NsqInputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)
	var wg sync.WaitGroup

	errChan := make(chan error, 1)
	retPackChan := make(chan *pipeline.PipelinePack, 1)
	defer close(errChan)
	defer close(retPackChan)

	c.Specify("A nsq input", func() {
		input := new(NsqInput)
		ith := new(plugins_ts.InputTestHelper)
		ith.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
		ith.MockInputRunner = pipelinemock.NewMockInputRunner(ctrl)

		config := input.ConfigStruct().(*NsqInputConfig)
		config.Topic = "test_topic"
		config.Channel = "test_channel"

		startInput := func() {
			wg.Add(1)
			go func() {
				errChan <- input.Run(ith.MockInputRunner, ith.MockHelper)
				wg.Done()
			}()
		}

		var mockConsumer *MockConsumer
		input.newConsumer = func(topic, channel string, config *nsq.Config) (Consumer, error) {
			mockConsumer, _ = NewMockConsumer(topic, channel, config)
			return mockConsumer, nil
		}

		ith.Pack = pipeline.NewPipelinePack(pConfig.InputRecycleChan())
		ith.PackSupply = make(chan *pipeline.PipelinePack, 1)
		ith.PackSupply <- ith.Pack

		inputName := "NsqInput"
		ith.MockInputRunner.EXPECT().Name().Return(inputName).AnyTimes()
		ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)

		c.Specify("that is started", func() {
			input.SetPipelineConfig(pConfig)

			c.Specify("gets messages its subscribed to", func() {
				c.Specify("injects messages into the pipeline when not configured with a decoder", func() {
					ith.MockInputRunner.EXPECT().Inject(ith.Pack).Do(func(p *pipeline.PipelinePack) {
						retPackChan <- p
					}).AnyTimes()

					err := input.Init(config)
					c.Expect(err, gs.IsNil)

					c.Expect(input.DecoderName, gs.Equals, "")

				})

				c.Specify("injects messages into the decoder when configured", func() {
					decoderName := "ScribbleDecoder"
					config.DecoderName = decoderName
					decoder := new(plugins.ScribbleDecoder)
					decoder.Init(&plugins.ScribbleDecoderConfig{})

					mockDecoderRunner := pipelinemock.NewMockDecoderRunner(ctrl)
					mockDecoderRunner.EXPECT().InChan().Return(retPackChan)
					ith.MockHelper.EXPECT().DecoderRunner(decoderName,
						fmt.Sprintf("%s-%s", inputName, decoderName),
					).Return(mockDecoderRunner, true)

					err := input.Init(config)
					c.Expect(err, gs.IsNil)

					c.Expect(input.DecoderName, gs.Equals, decoderName)
				})

				startInput()
				// Should get two finished conn calls since we connect
				// to lookupds and nsqds
				c.Expect(<-mockConsumer.finishedConn, gs.IsTrue)
				c.Expect(<-mockConsumer.finishedConn, gs.IsTrue)

				id := nsq.MessageID{}
				body := []byte("test")
				msg := nsq.NewMessage(id, body)
				for _, handler := range mockConsumer.handlers {
					handler.HandleMessage(msg)
				}

				p := <-retPackChan
				c.Expect(p.Message.GetPayload(), gs.Equals, "test")

			})

			input.Stop()
			wg.Wait()
			c.Expect(<-errChan, gs.IsNil)
		})

	})

}