Exemplo n.º 1
0
// LoadFromConfigFile loads a TOML configuration file and stores the
// result in the value pointed to by config. The maps in the config
// will be initialized as needed.
//
// The PipelineConfig should be already initialized before passed in via
// its Init function.
func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) {
	var configFile ConfigFile
	if _, err = toml.DecodeFile(filename, &configFile); err != nil {
		return fmt.Errorf("Error decoding config file: %s", err)
	}

	// Load all the plugins
	var errcnt uint
	for name, conf := range configFile {
		if name == "hekad" {
			continue
		}
		log.Printf("Loading: [%s]\n", name)
		errcnt += self.loadSection(name, conf)
	}

	// Add JSON/PROTOCOL_BUFFER decoders if none were configured
	var configDefault ConfigFile
	toml.Decode(defaultDecoderTOML, &configDefault)
	dWrappers := self.DecoderWrappers

	if _, ok := dWrappers["ProtobufDecoder"]; !ok {
		log.Println("Loading: [ProtobufDecoder]")
		errcnt += self.loadSection("ProtobufDecoder", configDefault["ProtobufDecoder"])
	}

	if errcnt != 0 {
		return fmt.Errorf("%d errors loading plugins", errcnt)
	}

	return
}
Exemplo n.º 2
0
func BenchmarkMultiDecodeProtobuf(b *testing.B) {
	b.StopTimer()
	pConfig := NewPipelineConfig(nil) // initializes Globals
	msg := pipeline_ts.GetTestMessage()
	msg.SetPayload("This is a test")
	pack := NewPipelinePack(pConfig.InputRecycleChan())
	pack.MsgBytes, _ = proto.Marshal(msg)
	decoder := new(MultiDecoder)
	decoder.SetPipelineConfig(pConfig)
	conf := decoder.ConfigStruct().(*MultiDecoderConfig)

	RegisterPlugin("ProtobufDecoder", func() interface{} {
		return &ProtobufDecoder{}
	})
	defer delete(AvailablePlugins, "ProtobufDecoder")
	var section PluginConfig
	_, err := toml.Decode("", &section)
	if err != nil {
		b.Fatalf("Error decoding empty TOML: %s", err.Error())
	}
	maker, err := NewPluginMaker("ProtobufDecoder", pConfig, section)
	if err != nil {
		b.Fatalf("Error decoding empty TOML: %s", err.Error())
	}
	pConfig.DecoderMakers["ProtobufDecoder"] = maker

	conf.CascadeStrategy = "first-wins"
	conf.Subs = []string{"sub"}
	decoder.Init(conf)

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		decoder.Decode(pack)
	}
}
Exemplo n.º 3
0
func TestConfigFile(t *testing.T) {
	var configFile ConfigFile
	if _, err := toml.Decode(configSource, &configFile); err != nil {
		t.Fatalf("Error decoding config: %s", err)
	}
	app, err := LoadApplication(configFile, env, 0)
	if err != nil {
		t.Fatalf("Error initializing app: %s", err)
	}
	defer app.Stop()
	hostname := "push.services.mozilla.com"
	if app.hostname != hostname {
		t.Errorf("Mismatched hostname: got %#v; want %#v", app.hostname, hostname)
	}
	origin, _ := url.ParseRequestURI("https://push.services.mozilla.com")
	origins := []*url.URL{origin}
	if !reflect.DeepEqual(origins, app.origins) {
		t.Errorf("Mismatched origins: got %#v; want %#v", app.origins, origins)
	}
	logger := app.Logger()
	if stdOutLogger, ok := logger.Logger.(*StdOutLogger); ok {
		if stdOutLogger.filter != 0 {
			t.Errorf("Mismatched log levels: got %#v; want 0", stdOutLogger.filter)
		}
		emitter := stdOutLogger.LogEmitter
		if _, ok := emitter.(*TextEmitter); !ok {
			t.Errorf("Log emitter type assertion failed: %#v", emitter)
		}
	} else {
		t.Errorf("Logger type assertion failed: %#v", logger)
	}
	store := app.Store()
	if noStore, ok := store.(*NoStore); ok {
		defaultChans := noStore.ConfigStruct().(*NoStoreConfig).MaxChannels
		if noStore.maxChannels != defaultChans {
			t.Errorf("Wrong default channel count for storage: got %d; want %d",
				noStore.maxChannels, defaultChans)
		}
	} else {
		t.Errorf("Storage type assertion failed: %#v", store)
	}
	pinger := app.PropPinger()
	if udpPing, ok := pinger.(*UDPPing); ok {
		if udpPing.app != app {
			t.Errorf("Wrong app instance for pinger: got %#v; want %#v",
				udpPing.app, app)
		}
		url := "http://push.services.mozilla.com/ping"
		if udpPing.config.URL != url {
			t.Errorf("Wrong pinger URL: got %#v; want %#v",
				udpPing.config.URL, url)
		}
	} else {
		t.Errorf("Pinger type assertion failed: %#v", pinger)
	}
}
Exemplo n.º 4
0
Arquivo: config.go Projeto: Nitro/heka
// PreloadFromConfigFile loads all plugin configuration from a TOML
// configuration file, generates a PluginMaker for each loaded section, and
// stores the created PluginMakers in the makersByCategory map. The
// PipelineConfig should be already initialized via the Init function before
// this method is called. PreloadFromConfigFile is not reentrant, so it should
// only be called serially, not from multiple concurrent goroutines.
func (self *PipelineConfig) PreloadFromConfigFile(filename string) error {
	var (
		configFile ConfigFile
		err        error
	)

	contents, err := ReplaceEnvsFile(filename)
	if err != nil {
		return err
	}

	if _, err = toml.Decode(contents, &configFile); err != nil {
		return fmt.Errorf("Error decoding config file: %s", err)
	}

	if self.makersByCategory == nil {
		self.makersByCategory = make(map[string][]PluginMaker)
	}

	if self.defaultConfigs == nil {
		self.defaultConfigs = makeDefaultConfigs()
	}

	// Load all the plugin makers and file them by category.
	for name, conf := range configFile {
		if name == HEKA_DAEMON {
			continue
		}
		if _, ok := self.defaultConfigs[name]; ok {
			self.defaultConfigs[name] = true
		}
		LogInfo.Printf("Pre-loading: [%s]\n", name)
		maker, err := NewPluginMaker(name, self, conf)
		if err != nil {
			self.log(err.Error())
			self.errcnt++
			continue
		}

		if maker.Type() == "MultiDecoder" {
			// Special case MultiDecoders so we can make sure they get
			// registered *after* all possible subdecoders.
			self.makersByCategory["MultiDecoder"] = append(
				self.makersByCategory["MultiDecoder"], maker)
		} else {
			category := maker.Category()
			self.makersByCategory[category] = append(
				self.makersByCategory[category], maker)
		}
	}
	return nil
}
Exemplo n.º 5
0
// Parses a Heka message and extracts the information necessary to start a new
// SandboxFilter
func (this *SandboxManagerFilter) loadSandbox(fr pipeline.FilterRunner,
	h pipeline.PluginHelper, dir string, msg *message.Message) (err error) {
	fv, _ := msg.GetFieldValue("config")
	if config, ok := fv.(string); ok {
		var configFile pipeline.ConfigFile
		if _, err = toml.Decode(config, &configFile); err != nil {
			return fmt.Errorf("loadSandbox failed: %s\n", err)
		} else {
			for name, conf := range configFile {
				name = getSandboxName(fr.Name(), name)
				if _, ok := h.Filter(name); ok {
					// todo support reload
					return fmt.Errorf("loadSandbox failed: %s is already running", name)
				}
				fr.LogMessage(fmt.Sprintf("Loading: %s", name))
				confFile := filepath.Join(dir, fmt.Sprintf("%s.toml", name))
				err = ioutil.WriteFile(confFile, []byte(config), 0600)
				if err != nil {
					return
				}
				var sbc SandboxConfig
				if err = toml.PrimitiveDecode(conf, &sbc); err != nil {
					return fmt.Errorf("loadSandbox failed: %s\n", err)
				}
				scriptFile := filepath.Join(dir, fmt.Sprintf("%s.%s", name, sbc.ScriptType))
				err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
				if err != nil {
					removeAll(dir, fmt.Sprintf("%s.*", name))
					return
				}
				// check/clear the old state preservation file
				// this avoids issues with changes to the data model since the last load
				// and prevents holes in the graph from looking like anomalies
				os.Remove(filepath.Join(pipeline.PrependBaseDir(DATA_DIR), name+DATA_EXT))
				var runner pipeline.FilterRunner
				runner, err = this.createRunner(dir, name, conf)
				if err != nil {
					removeAll(dir, fmt.Sprintf("%s.*", name))
					return
				}
				err = h.PipelineConfig().AddFilterRunner(runner)
				if err == nil {
					this.currentFilters++
				}
				break // only interested in the first item
			}
		}
	}
	return
}
Exemplo n.º 6
0
// Parses a Heka message and extracts the information necessary to start a new
// SandboxFilter
func (this *SandboxManagerFilter) loadSandbox(fr pipeline.FilterRunner,
	h pipeline.PluginHelper, dir string, msg *message.Message) (err error) {

	fv, _ := msg.GetFieldValue("config")
	if config, ok := fv.(string); ok {
		var configFile pipeline.ConfigFile
		if _, err = toml.Decode(config, &configFile); err != nil {
			return fmt.Errorf("loadSandbox failed: %s\n", err)
		}

		for name, conf := range configFile {
			name = getSandboxName(fr.Name(), name)
			if _, ok := h.Filter(name); ok {
				// todo support reload
				return fmt.Errorf("loadSandbox failed: %s is already running", name)
			}
			fr.LogMessage(fmt.Sprintf("Loading: %s", name))
			confFile := filepath.Join(dir, fmt.Sprintf("%s.toml", name))
			err = ioutil.WriteFile(confFile, []byte(config), 0600)
			if err != nil {
				return
			}
			var sbc SandboxConfig
			// Default, will get overwritten if necessary
			sbc.ScriptType = "lua"
			if err = toml.PrimitiveDecode(conf, &sbc); err != nil {
				return fmt.Errorf("loadSandbox failed: %s\n", err)
			}
			scriptFile := filepath.Join(dir, fmt.Sprintf("%s.%s", name, sbc.ScriptType))
			err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
			if err != nil {
				removeAll(dir, fmt.Sprintf("%s.*", name))
				return
			}
			var runner pipeline.FilterRunner
			runner, err = this.createRunner(dir, name, conf)
			if err != nil {
				removeAll(dir, fmt.Sprintf("%s.*", name))
				return
			}
			err = this.pConfig.AddFilterRunner(runner)
			if err == nil {
				atomic.AddInt32(&this.currentFilters, 1)
			}
			break // only interested in the first item
		}
	}
	return
}
Exemplo n.º 7
0
// LoadFromConfigFile loads a TOML configuration file and stores the
// result in the value pointed to by config. The maps in the config
// will be initialized as needed.
//
// The PipelineConfig should be already initialized before passed in via
// its Init function.
func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) {
	var configFile ConfigFile
	if _, err = toml.DecodeFile(filename, &configFile); err != nil {
		return fmt.Errorf("Error decoding config file: %s", err)
	}

	// Load all the plugins
	var errcnt uint
	for name, conf := range configFile {
		log.Println("Loading: ", name)
		errcnt += self.loadSection(name, conf)
	}

	// Add JSON/PROTOCOL_BUFFER decoders if none were configured
	var configDefault ConfigFile
	toml.Decode(defaultDecoderTOML, &configDefault)
	dWrappers := self.DecoderWrappers

	if _, ok := dWrappers["JsonDecoder"]; !ok {
		log.Println("Loading: JsonDecoder")
		errcnt += self.loadSection("JsonDecoder", configDefault["JsonDecoder"])
	}
	if _, ok := dWrappers["ProtobufDecoder"]; !ok {
		log.Println("Loading: ProtobufDecoder")
		errcnt += self.loadSection("ProtobufDecoder", configDefault["ProtobufDecoder"])
	}

	// Create / prep the DecoderSet pool
	var dRunner DecoderRunner
	for i := 0; i < Globals().DecoderPoolSize; i++ {
		if self.DecoderSets[i], err = newDecoderSet(dWrappers); err != nil {
			log.Println(err)
			errcnt += 1
		}
		for _, dRunner = range self.DecoderSets[i].AllByName() {
			dRunner.Start(self, &self.decodersWg)
		}
		self.decodersChan <- self.DecoderSets[i]
	}

	if errcnt != 0 {
		return fmt.Errorf("%d errors loading plugins", errcnt)
	}

	return
}
Exemplo n.º 8
0
func (this *SandboxManagerFilter) loadSandbox(fr FilterRunner,
	h PluginHelper, dir string, msg *message.Message) (err error) {
	fv, _ := msg.GetFieldValue("config")
	if config, ok := fv.(string); ok {
		var configFile ConfigFile
		if _, err = toml.Decode(config, &configFile); err != nil {
			return fmt.Errorf("loadSandbox failed: %s\n", err)
		} else {
			for name, conf := range configFile {
				name = getSandboxName(fr.Name(), name)
				if _, ok := h.Filter(name); ok {
					// todo support reload
					return fmt.Errorf("loadSandbox failed: %s is already running", name)
				}
				fr.LogMessage(fmt.Sprintf("Loading: %s", name))
				confFile := path.Join(dir, fmt.Sprintf("%s.toml", name))
				err = ioutil.WriteFile(confFile, []byte(config), 0600)
				if err != nil {
					return
				}
				var sbfc SandboxFilterConfig
				if err = toml.PrimitiveDecode(conf, &sbfc); err != nil {
					return fmt.Errorf("loadSandbox failed: %s\n", err)
				}
				scriptFile := path.Join(dir, fmt.Sprintf("%s.%s", name, sbfc.Sbc.ScriptType))
				err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
				if err != nil {
					removeAll(dir, fmt.Sprintf("%s.*", name))
					return
				}
				var runner FilterRunner
				runner, err = createRunner(dir, name, conf)
				if err != nil {
					removeAll(dir, fmt.Sprintf("%s.*", name))
					return
				}
				err = h.PipelineConfig().AddFilterRunner(runner)
				if err == nil {
					this.currentFilters++
				}
				break // only interested in the first item
			}
		}
	}
	return
}
Exemplo n.º 9
0
Arquivo: config.go Projeto: Nitro/heka
func (self *PipelineConfig) RegisterDefault(name string) error {
	var config ConfigFile
	confStr := fmt.Sprintf("[%s]", name)
	toml.Decode(confStr, &config)
	LogInfo.Printf("Pre-loading: %s\n", confStr)
	maker, err := NewPluginMaker(name, self, config[name])
	if err != nil {
		// This really shouldn't happen.
		return err
	}
	LogInfo.Printf("Loading: [%s]\n", maker.Name())
	if _, err = maker.PrepConfig(); err != nil {
		return err
	}
	category := maker.Category()
	self.makersLock.Lock()
	self.makers[category][name] = maker
	self.makersLock.Unlock()
	// If we ever add a default input, filter, or output we'd need to call
	// maker.MakeRunner() here and store the runner on the PipelineConfig.
	return nil
}
Exemplo n.º 10
0
func LoadHekadConfig(configPath string) (config *HekadConfig, err error) {
	idle, _ := time.ParseDuration("2m")

	config = &HekadConfig{Maxprocs: 1,
		PoolSize:              100,
		ChanSize:              50,
		CpuProfName:           "",
		MemProfName:           "",
		MaxMsgLoops:           4,
		MaxMsgProcessInject:   1,
		MaxMsgProcessDuration: 100000,
		MaxMsgTimerInject:     10,
		MaxPackIdle:           idle,
		BaseDir:               filepath.FromSlash("/var/cache/hekad"),
		ShareDir:              filepath.FromSlash("/usr/share/heka"),
		SampleDenominator:     1000,
		PidFile:               "",
	}

	var configFile map[string]toml.Primitive
	p, err := os.Open(configPath)
	if err != nil {
		return nil, fmt.Errorf("Error opening config file: %s", err)
	}
	fi, err := p.Stat()
	if err != nil {
		return nil, fmt.Errorf("Error fetching config file info: %s", err)
	}

	if fi.IsDir() {
		files, _ := ioutil.ReadDir(configPath)
		for _, f := range files {
			fName := f.Name()
			if !strings.HasSuffix(fName, ".toml") {
				// Skip non *.toml files in a config dir.
				continue
			}
			fPath := filepath.Join(configPath, fName)
			contents, err := pipeline.ReplaceEnvsFile(fPath)
			if err != nil {
				return nil, err
			}
			if _, err = toml.Decode(contents, &configFile); err != nil {
				return nil, fmt.Errorf("Error decoding config file: %s", err)
			}
		}
	} else {
		contents, err := pipeline.ReplaceEnvsFile(configPath)
		if err != nil {
			return nil, err
		}
		if _, err = toml.Decode(contents, &configFile); err != nil {
			return nil, fmt.Errorf("Error decoding config file: %s", err)
		}
	}

	empty_ignore := map[string]interface{}{}
	parsed_config, ok := configFile[pipeline.HEKA_DAEMON]
	if ok {
		if err = toml.PrimitiveDecodeStrict(parsed_config, config, empty_ignore); err != nil {
			err = fmt.Errorf("Can't unmarshal config: %s", err)
		}
	}

	return
}
Exemplo n.º 11
0
func MultiDecoderSpec(c gospec.Context) {
	t := &pipeline_ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := NewPipelineConfig(nil) // initializes Globals()

	c.Specify("A MultiDecoder", func() {
		subsTOML := `[StartsWithM]
		type = "PayloadRegexDecoder"
		match_regex = '^(?P<TheData>m.*)'
		log_errors = true
		[StartsWithM.message_fields]
		StartsWithM = "%TheData%"
	
		[StartsWithS]
		type = "PayloadRegexDecoder"
		match_regex = '^(?P<TheData>s.*)'
		log_errors = true
		[StartsWithS.message_fields]
		StartsWithS = "%TheData%"
	
		[StartsWithM2]
		type = "PayloadRegexDecoder"
		match_regex = '^(?P<TheData>m.*)'
		log_errors = true
		[StartsWithM2.message_fields]
		StartsWithM2 = "%TheData%"
		`

		RegisterPlugin("PayloadRegexDecoder", func() interface{} {
			return &PayloadRegexDecoder{}
		})
		defer delete(AvailablePlugins, "PayloadRegexDecoder")

		var configFile ConfigFile
		_, err := toml.Decode(subsTOML, &configFile)
		c.Assume(err, gs.IsNil)

		decoder := new(MultiDecoder)
		decoder.SetName("MyMultiDecoder")
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*MultiDecoderConfig)

		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		mSection, ok := configFile["StartsWithM"]
		c.Assume(ok, gs.IsTrue)
		mMaker, err := NewPluginMaker("StartsWithM", pConfig, mSection)
		c.Assume(err, gs.IsNil)
		pConfig.DecoderMakers["StartsWithM"] = mMaker

		conf.Subs = []string{"StartsWithM"}
		errMsg := "All subdecoders failed."

		dRunner := pipelinemock.NewMockDecoderRunner(ctrl)
		// An error will be spit out b/c there's no real *dRunner in there;
		// doesn't impact the tests.
		dRunner.EXPECT().LogError(gomock.Any())

		c.Specify("decodes simple messages", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "matching text"
			pack.Message.SetPayload(regex_data)
			_, err = decoder.Decode(pack)
			c.Assume(err, gs.IsNil)

			value, ok := pack.Message.GetFieldValue("StartsWithM")
			c.Assume(ok, gs.IsTrue)
			c.Expect(value, gs.Equals, regex_data)
		})

		c.Specify("returns an error if all decoders fail", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "non-matching text"
			pack.Message.SetPayload(regex_data)
			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, errMsg)
		})

		c.Specify("logs subdecoder failures when configured to do so", func() {
			conf.LogSubErrors = true
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			decoder.SetDecoderRunner(dRunner)
			regex_data := "non-matching text"
			pack.Message.SetPayload(regex_data)

			// Expect that we log an error for undecoded message.
			dRunner.EXPECT().LogError(fmt.Errorf(
				"Subdecoder 'StartsWithM' decode error: No match: %s", regex_data)).AnyTimes()

			packs, err := decoder.Decode(pack)
			c.Expect(len(packs), gs.Equals, 0)
			c.Expect(err.Error(), gs.Equals, errMsg)
		})

		c.Specify("sets subdecoder runner correctly", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			// Call LogError to appease the angry gomock gods.
			dRunner.LogError(errors.New("foo"))

			// Now create a real *dRunner, pass it in, make sure a wrapper
			// gets handed to the subdecoder.
			dr := NewDecoderRunner(decoder.Name, decoder, 10)
			decoder.SetDecoderRunner(dr)
			sub := decoder.Decoders[0]
			subRunner := sub.(*PayloadRegexDecoder).dRunner
			c.Expect(subRunner.Name(), gs.Equals,
				fmt.Sprintf("%s-StartsWithM", decoder.Name))
			c.Expect(subRunner.Decoder(), gs.Equals, sub)
		})

		c.Specify("with multiple registered decoders", func() {
			sSection, ok := configFile["StartsWithS"]
			c.Assume(ok, gs.IsTrue)
			sMaker, err := NewPluginMaker("StartsWithS", pConfig, sSection)
			c.Assume(err, gs.IsNil)
			pConfig.DecoderMakers["StartsWithS"] = sMaker

			m2Section, ok := configFile["StartsWithM2"]
			c.Assume(ok, gs.IsTrue)
			m2Maker, err := NewPluginMaker("StartsWithM2", pConfig, m2Section)
			c.Assume(err, gs.IsNil)
			pConfig.DecoderMakers["StartsWithM2"] = m2Maker

			conf.Subs = append(conf.Subs, "StartsWithS", "StartsWithM2")

			// Two more subdecoders means two more LogError calls.
			dRunner.EXPECT().LogError(gomock.Any()).Times(2)

			c.Specify("defaults to `first-wins` cascading", func() {
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)

				c.Specify("on a first match condition", func() {
					pack.Message.SetPayload("match first")
					_, err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("and a second match condition", func() {
					pack.Message.SetPayload("second match")
					_, err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("returning an error if they all fail", func() {
					pack.Message.SetPayload("won't match")
					packs, err := decoder.Decode(pack)
					c.Expect(len(packs), gs.Equals, 0)
					c.Expect(err.Error(), gs.Equals, errMsg)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})
			})

			c.Specify("and using `all` cascading", func() {
				conf.CascadeStrategy = "all"
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)

				c.Specify("matches multiples when appropriate", func() {
					pack.Message.SetPayload("matches twice")
					_, err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsTrue)
				})

				c.Specify("matches singles when appropriate", func() {
					pack.Message.SetPayload("second match")
					_, err = decoder.Decode(pack)
					c.Expect(err, gs.IsNil)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsTrue)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})

				c.Specify("returns an error if they all fail", func() {
					pack.Message.SetPayload("won't match")
					packs, err := decoder.Decode(pack)
					c.Expect(len(packs), gs.Equals, 0)
					c.Expect(err.Error(), gs.Equals, errMsg)
					_, ok = pack.Message.GetFieldValue("StartsWithM")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithS")
					c.Expect(ok, gs.IsFalse)
					_, ok = pack.Message.GetFieldValue("StartsWithM2")
					c.Expect(ok, gs.IsFalse)
				})
			})
		})
	})

	c.Specify("A MultiDecoder w/ MultiOutput", func() {
		subsTOML := `[sub0]
		type = "MultiOutputDecoder"

		[sub1]
		type = "MultiOutputDecoder"

		[sub2]
		type = "MultiOutputDecoder"
		`

		RegisterPlugin("MultiOutputDecoder", func() interface{} {
			return &MultiOutputDecoder{}
		})
		defer delete(AvailablePlugins, "MultiOutputDecoder")

		var configFile ConfigFile
		_, err := toml.Decode(subsTOML, &configFile)

		decoder := new(MultiDecoder)
		decoder.SetName("MyMultiDecoder")
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*MultiDecoderConfig)
		conf.CascadeStrategy = "all"

		supply := make(chan *PipelinePack, 10)
		pack := NewPipelinePack(supply)

		sub0Section, ok := configFile["sub0"]
		c.Assume(ok, gs.IsTrue)
		sub0Maker, err := NewPluginMaker("sub0", pConfig, sub0Section)
		c.Assume(err, gs.IsNil)
		pConfig.DecoderMakers["sub0"] = sub0Maker

		sub1Section, ok := configFile["sub1"]
		c.Assume(ok, gs.IsTrue)
		sub1Maker, err := NewPluginMaker("sub1", pConfig, sub1Section)
		c.Assume(err, gs.IsNil)
		pConfig.DecoderMakers["sub1"] = sub1Maker

		sub2Section, ok := configFile["sub2"]
		c.Assume(ok, gs.IsTrue)
		sub2Maker, err := NewPluginMaker("sub2", pConfig, sub2Section)
		c.Assume(err, gs.IsNil)
		pConfig.DecoderMakers["sub2"] = sub2Maker

		conf.Subs = []string{"sub0", "sub1", "sub2"}
		dRunner := pipelinemock.NewMockDecoderRunner(ctrl)

		c.Specify("always tries to decode all packs", func() {
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)
			decoder.SetDecoderRunner(dRunner)

			packs, err := decoder.Decode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(len(packs), gs.Equals, 15)
		})
	})
}
Exemplo n.º 12
0
func LoadConfig(configPath *string) *GlobalConfig {
	f, err := os.Open(*configPath)
	if err != nil {
		log.Fatalln("open config file or dir failed: ", *configPath)
	}
	defer f.Close()

	fs, err := os.Stat(*configPath)
	if err != nil {
		log.Fatalln("get stat of file or dir failed: ", *configPath)
	}

	if fs.IsDir() {
		log.Fatalln("config file must be file: ", *configPath)
	}

	if !strings.HasSuffix(*configPath, ".toml") {
		log.Fatalln("config file must has .toml suffix")
	}

	data, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatalln("ioutil.ReadAll config file failed: ", *configPath)
	}

	// log.Println(data)

	globals := &GlobalConfig{}
	_, err = toml.Decode(string(data), &cfgs)
	if err != nil {
		log.Fatalln("toml.Decode data failed: ", err)
	}
	log.Println(cfgs)
	// log.Println(cfgs)

	// kafkacfg := &KafkaInputConfig{}
	base := &BaseConfig{}
	kafkacfg := NewKafkaInputConfig()
	influxcfg := NewInfluxDBOutputConfig()

	empty_ignore := map[string]interface{}{}

	parsed_globals, ok := cfgs["global"]

	if !ok {
		log.Fatalln("global base toml must be set")
	}
	if err = toml.PrimitiveDecodeStrict(parsed_globals, base, empty_ignore); err != nil {
		log.Fatalln("global base decode failed: ", err)
	}

	globals.Base = base

	log.Println(globals.Base)

	parsed_config, ok := cfgs[globals.Base.Input]
	if ok {
		if err = toml.PrimitiveDecodeStrict(parsed_config, kafkacfg, empty_ignore); err != nil {
			// err = fmt.Errorf("Can't unmarshal config: %s", err)
			log.Fatalln("can't unmarshal config: ", err)
		}
		// log.Println(kafkacfg)
		globals.KafkaConfig = kafkacfg
	}
	parsed_config_influxdb, ok := cfgs[globals.Base.Output]
	if ok {
		if err = toml.PrimitiveDecodeStrict(parsed_config_influxdb, influxcfg, empty_ignore); err != nil {
			// err = fmt.Errorf("Can't unmarshal config: %s", err)
			log.Fatalln("can't unmarshal config: ", err)
		}
		// log.Println(kafkacfg)
		globals.InfluxDBConfig = influxcfg
	}

	return globals
}
Exemplo n.º 13
0
func QueueBufferSpec(c gs.Context) {
	tmpDir, tmpErr := ioutil.TempDir("", "queuebuffer-tests")

	defer func() {
		tmpErr = os.RemoveAll(tmpDir)
		c.Expect(tmpErr, gs.Equals, nil)
	}()

	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	c.Specify("QueueBuffer Internals", func() {
		pConfig := NewPipelineConfig(nil)

		err := pConfig.RegisterDefault("HekaFramingSplitter")
		c.Assume(err, gs.IsNil)
		RegisterPlugin("FooOutput", func() interface{} {
			return &FooOutput{}
		})

		outputToml := `[FooOutput]
        message_matcher = "TRUE"
        `

		var configFile ConfigFile
		_, err = toml.Decode(outputToml, &configFile)
		c.Assume(err, gs.IsNil)
		section, ok := configFile["FooOutput"]
		c.Assume(ok, gs.IsTrue)
		maker, err := NewPluginMaker("FooOutput", pConfig, section)
		c.Assume(err, gs.IsNil)

		orTmp, err := maker.MakeRunner("FooOutput")
		c.Assume(err, gs.IsNil)
		or := orTmp.(*foRunner)

		// h := NewMockPluginHelper(ctrl)
		// h.EXPECT().PipelineConfig().Return(pConfig)

		qConfig := &QueueBufferConfig{
			FullAction:        "block",
			CursorUpdateCount: 1,
			MaxFileSize:       66000,
		}
		feeder, reader, err := NewBufferSet(tmpDir, "test", qConfig, or, pConfig)
		// bufferedOutput, err := NewBufferedOutput(tmpDir, "test", or, h, uint64(0))
		c.Assume(err, gs.IsNil)
		msg := ts.GetTestMessage()

		c.Specify("fileExists", func() {
			c.Expect(fileExists(tmpDir), gs.IsTrue)
			c.Expect(fileExists(filepath.Join(tmpDir, "test.log")), gs.IsFalse)
		})

		c.Specify("extractBufferId", func() {
			id, err := extractBufferId("555.log")
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(555))
			id, err = extractBufferId("")
			c.Expect(err, gs.Not(gs.IsNil))
			id, err = extractBufferId("a.log")
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("findBufferId", func() {
			c.Expect(findBufferId(tmpDir, true), gs.Equals, uint(0))
			c.Expect(findBufferId(tmpDir, false), gs.Equals, uint(0))
			fd, err := os.Create(filepath.Join(tmpDir, "4.log"))
			c.Expect(err, gs.IsNil)
			fd.Close()
			fd, err = os.Create(filepath.Join(tmpDir, "5.log"))
			c.Expect(err, gs.IsNil)
			fd.Close()
			fd, err = os.Create(filepath.Join(tmpDir, "6a.log"))
			c.Expect(err, gs.IsNil)
			fd.Close()
			c.Expect(findBufferId(tmpDir, false), gs.Equals, uint(4))
			c.Expect(findBufferId(tmpDir, true), gs.Equals, uint(5))
		})

		c.Specify("writeCheckpoint", func() {
			reader.checkpointFilename = filepath.Join(tmpDir, "cp.txt")
			err := reader.writeCheckpoint("43 99999")
			c.Expect(err, gs.IsNil)
			c.Expect(fileExists(reader.checkpointFilename), gs.IsTrue)

			id, offset, err := readCheckpoint(reader.checkpointFilename)
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(43))
			c.Expect(offset, gs.Equals, int64(99999))

			err = reader.writeCheckpoint("43 1")
			c.Expect(err, gs.IsNil)
			id, offset, err = readCheckpoint(reader.checkpointFilename)
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(43))
			c.Expect(offset, gs.Equals, int64(1))
			reader.checkpointFile.Close()
		})

		c.Specify("readCheckpoint", func() {
			cp := filepath.Join(tmpDir, "cp.txt")
			file, err := os.OpenFile(cp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
			c.Expect(err, gs.IsNil)
			id, offset, err := readCheckpoint(cp)
			c.Expect(err, gs.Not(gs.IsNil))

			file.WriteString("22")
			id, offset, err = readCheckpoint(cp)
			c.Expect(err.Error(), gs.Equals, "invalid checkpoint format")

			file.Seek(0, 0)
			file.WriteString("aa 22")
			id, offset, err = readCheckpoint(cp)
			c.Expect(err.Error(), gs.Equals, "invalid checkpoint id")

			file.Seek(0, 0)
			file.WriteString("43 aa")
			id, offset, err = readCheckpoint(cp)
			c.Expect(err.Error(), gs.Equals, "invalid checkpoint offset")

			file.Seek(0, 0)
			file.WriteString("43 22")
			file.Close()

			id, offset, err = readCheckpoint(cp)
			c.Expect(err, gs.IsNil)
			c.Expect(id, gs.Equals, uint(43))
			c.Expect(offset, gs.Equals, int64(22))
		})

		c.Specify("RollQueue", func() {
			feeder.queue = tmpDir
			err := feeder.RollQueue()
			c.Expect(err, gs.IsNil)
			c.Expect(fileExists(getQueueFilename(feeder.queue,
				feeder.writeId)), gs.IsTrue)
			feeder.writeFile.WriteString("this is a test item")
			feeder.writeFile.Close()

			reader.checkpointFilename = filepath.Join(tmpDir, "cp.txt")
			reader.queue = tmpDir
			reader.writeCheckpoint(fmt.Sprintf("%d 10", feeder.writeId))
			reader.checkpointFile.Close()
			err = reader.initReadFile()
			c.Assume(err, gs.IsNil)
			buf := make([]byte, 4)
			n, err := reader.readFile.Read(buf)
			c.Expect(n, gs.Equals, 4)
			c.Expect(string(buf), gs.Equals, "test")
			feeder.writeFile.Close()
			reader.readFile.Close()
		})

		c.Specify("QueueRecord", func() {
			reader.checkpointFilename = filepath.Join(tmpDir, "cp.txt")
			reader.queue = tmpDir
			newpack := NewPipelinePack(nil)
			newpack.Message = msg
			payload := "Write me out to the network"
			newpack.Message.SetPayload(payload)
			encoder := client.NewProtobufEncoder(nil)
			protoBytes, err := encoder.EncodeMessage(newpack.Message)
			newpack.MsgBytes = protoBytes
			expectedLen := 115

			c.Specify("adds framing", func() {
				err = feeder.RollQueue()
				c.Expect(err, gs.IsNil)
				err = feeder.QueueRecord(newpack)
				fName := getQueueFilename(feeder.queue, feeder.writeId)
				c.Expect(fileExists(fName), gs.IsTrue)
				c.Expect(err, gs.IsNil)
				feeder.writeFile.Close()

				f, err := os.Open(fName)
				c.Expect(err, gs.IsNil)

				n, record, err := reader.sRunner.GetRecordFromStream(f)
				f.Close()
				c.Expect(n, gs.Equals, expectedLen)
				c.Expect(err, gs.IsNil)
				headerLen := int(record[1]) + message.HEADER_FRAMING_SIZE
				record = record[headerLen:]
				outMsg := new(message.Message)
				proto.Unmarshal(record, outMsg)
				c.Expect(outMsg.GetPayload(), gs.Equals, payload)
			})

			c.Specify("when queue has limit", func() {
				feeder.Config.MaxBufferSize = uint64(200)
				c.Expect(feeder.queueSize.Get(), gs.Equals, uint64(0))

				err = feeder.RollQueue()
				c.Expect(err, gs.IsNil)

				err = feeder.QueueRecord(newpack)
				c.Expect(err, gs.IsNil)
				c.Expect(feeder.queueSize.Get(), gs.Equals, uint64(expectedLen))
			})

			c.Specify("when queue has limit and is full", func() {
				feeder.Config.MaxBufferSize = uint64(50)
				feeder.Config.MaxFileSize = uint64(50)

				c.Expect(feeder.queueSize.Get(), gs.Equals, uint64(0))
				err = feeder.RollQueue()
				c.Expect(err, gs.IsNil)
				queueFiles, err := ioutil.ReadDir(feeder.queue)
				c.Expect(err, gs.IsNil)
				numFiles := len(queueFiles)

				err = feeder.QueueRecord(newpack)
				c.Expect(err, gs.Equals, QueueIsFull)
				c.Expect(feeder.queueSize.Get(), gs.Equals, uint64(0))

				// Bump the max queue size so it will accept a record.
				feeder.Config.MaxBufferSize = uint64(120)
				err = feeder.QueueRecord(newpack)
				c.Expect(err, gs.IsNil)

				// Queue should have rolled.
				queueFiles, err = ioutil.ReadDir(feeder.queue)
				c.Expect(err, gs.IsNil)
				c.Expect(len(queueFiles), gs.Equals, numFiles+1)

				// Try to queue one last time, it should fail.
				err = feeder.QueueRecord(newpack)
				c.Expect(err, gs.Equals, QueueIsFull)

				// Ensure queue didn't roll twice.
				queueFiles, err = ioutil.ReadDir(feeder.queue)
				c.Expect(err, gs.IsNil)
				c.Expect(len(queueFiles), gs.Equals, numFiles+1)
			})

			c.Specify("rolls when queue file hits max size", func() {
				feeder.Config.MaxFileSize = uint64(300)
				c.Assume(feeder.writeFileSize, gs.Equals, uint64(0))

				// First two shouldn't trigger roll.
				err = feeder.QueueRecord(newpack)
				c.Expect(err, gs.IsNil)
				err = feeder.QueueRecord(newpack)
				c.Expect(err, gs.IsNil)
				c.Expect(feeder.writeFileSize, gs.Equals, uint64(expectedLen*2))
				queueFiles, err := ioutil.ReadDir(feeder.queue)
				c.Expect(err, gs.IsNil)
				c.Expect(len(queueFiles), gs.Equals, 1)

				// Third one should.
				err = feeder.QueueRecord(newpack)
				c.Expect(err, gs.IsNil)
				c.Expect(feeder.writeFileSize, gs.Equals, uint64(expectedLen))
				queueFiles, err = ioutil.ReadDir(feeder.queue)
				c.Expect(err, gs.IsNil)
				c.Expect(len(queueFiles), gs.Equals, 2)
			})
		})

		c.Specify("getQueueBufferSize", func() {
			c.Expect(getQueueBufferSize(tmpDir), gs.Equals, uint64(0))

			fd, _ := os.Create(filepath.Join(tmpDir, "4.log"))
			fd.WriteString("0123456789")
			fd.Close()

			fd, _ = os.Create(filepath.Join(tmpDir, "5.log"))
			fd.WriteString("0123456789")
			fd.Close()

			// Only size of *.log files should be taken in calculations.
			fd, _ = os.Create(filepath.Join(tmpDir, "random_file"))
			fd.WriteString("0123456789")
			fd.Close()

			c.Expect(getQueueBufferSize(tmpDir), gs.Equals, uint64(20))
		})
	})
}
Exemplo n.º 14
0
// Loads all plugin configuration from a TOML configuration file. The
// PipelineConfig should be already initialized via the Init function before
// this method is called.
func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) {
	var configFile ConfigFile

	if _, err = toml.DecodeFile(filename, &configFile); err != nil {
		return fmt.Errorf("Error decoding config file: %s", err)
	}

	var (
		errcnt              uint
		protobufDRegistered bool
		protobufERegistered bool
	)
	sectionsByCategory := make(map[string][]*ConfigSection)

	// Load all the plugin globals and file them by category.
	for name, conf := range configFile {
		if name == HEKA_DAEMON {
			continue
		}
		log.Printf("Pre-loading: [%s]\n", name)
		section := &ConfigSection{
			name:        name,
			tomlSection: conf,
		}
		if err = self.loadPluginGlobals(section); err != nil {
			self.log(err.Error())
			errcnt++
			continue
		}

		category := getPluginCategory(section.globals.Typ)
		if category == "" {
			self.log(fmt.Sprintf("Type doesn't contain valid plugin name: %s\n",
				section.globals.Typ))
			errcnt++
			continue
		}
		section.category = category
		if section.globals.Typ == "MultiDecoder" {
			// Special case MultiDecoders so we can make sure they get
			// registered *after* all possible subdecoders.
			sectionsByCategory["MultiDecoder"] = append(sectionsByCategory["MultiDecoder"],
				section)
		} else {
			sectionsByCategory[category] = append(sectionsByCategory[category], section)
		}
		if name == "ProtobufDecoder" {
			protobufDRegistered = true
		}
		if name == "ProtobufEncoder" {
			protobufERegistered = true
		}
	}

	// Make sure ProtobufDecoder is registered.
	if !protobufDRegistered {
		var configDefault ConfigFile
		toml.Decode(protobufDecoderToml, &configDefault)
		log.Println("Pre-loading: [ProtobufDecoder]")
		section := &ConfigSection{
			name:        "ProtobufDecoder",
			category:    "Decoder",
			tomlSection: configDefault["ProtobufDecoder"],
		}
		if err = self.loadPluginGlobals(section); err != nil {
			// This really shouldn't happen.
			self.log(err.Error())
			errcnt++
		} else {
			sectionsByCategory["Decoder"] = append(sectionsByCategory["Decoder"],
				section)
		}
	}

	// Make sure ProtobufEncoder is registered.
	if !protobufERegistered {
		var configDefault ConfigFile
		toml.Decode(protobufEncoderToml, &configDefault)
		log.Println("Pre-loading: [ProtobufEncoder]")
		section := &ConfigSection{
			name:        "ProtobufEncoder",
			category:    "Encoder",
			tomlSection: configDefault["ProtobufEncoder"],
		}
		if err = self.loadPluginGlobals(section); err != nil {
			// This really shouldn't happen.
			self.log(err.Error())
			errcnt++
		} else {
			sectionsByCategory["Encoder"] = append(sectionsByCategory["Encoder"],
				section)
		}
	}

	multiDecoders := make([]multiDecoderNode, len(sectionsByCategory["MultiDecoder"]))
	multiConfigs := make(map[string]*ConfigSection)

	for i, section := range sectionsByCategory["MultiDecoder"] {

		multiConfigs[section.name] = section

		multiDecoders[i] = newMultiDecoderNode(section.name, subsFromSection(section.tomlSection))

	}
	multiDecoders, err = orderDependencies(multiDecoders)
	if err != nil {
		return err
	}
	for i, d := range multiDecoders {
		sectionsByCategory["MultiDecoder"][i] = multiConfigs[d.name]
	}

	// Append MultiDecoders to the end of the Decoders list.
	sectionsByCategory["Decoder"] = append(sectionsByCategory["Decoder"],
		sectionsByCategory["MultiDecoder"]...)

	// Force decoders and encoders to be registered before the other plugin
	// types are initialized so we know they'll be there for inputs and
	// outputs to use during initialization.
	order := []string{"Decoder", "Encoder", "Input", "Filter", "Output"}
	for _, category := range order {
		for _, section := range sectionsByCategory[category] {
			log.Printf("Loading: [%s]\n", section.name)
			if err = self.loadSection(section); err != nil {
				self.log(err.Error())
				errcnt++
			}
		}
	}

	if errcnt != 0 {
		return fmt.Errorf("%d errors loading plugins", errcnt)
	}

	return
}