Esempio n. 1
0
func LoadHekadConfig(filename string) (config *HekadConfig, err error) {
	config = &HekadConfig{Maxprocs: 1,
		PoolSize:            100,
		DecoderPoolSize:     4,
		ChanSize:            50,
		CpuProfName:         "",
		MemProfName:         "",
		MaxMsgLoops:         4,
		MaxMsgProcessInject: 1,
		MaxMsgTimerInject:   10,
	}

	var configFile map[string]toml.Primitive

	if _, err = toml.DecodeFile(filename, &configFile); err != nil {
		return nil, fmt.Errorf("Error decoding config file: %s", err)
	}

	empty_ignore := map[string]interface{}{}
	parsed_config, ok := configFile["hekad"]
	if ok {
		if err = toml.PrimitiveDecodeStrict(parsed_config, &config, empty_ignore); err != nil {
			err = fmt.Errorf("Can't unmarshal config: %s", err)
		}
	}

	return
}
Esempio n. 2
0
// PrepConfig generates a config struct for the plugin (instantiating an
// instance of the plugin to do so, if necessary) and decodes the TOML config
// into the generated struct.
func (m *pluginMaker) PrepConfig() error {
	if m.configPrepped {
		// Already done, just return.
		return nil
	}

	if m.configStruct == nil {
		m.makeConfig()
	} else if m.plugin == nil {
		m.plugin = m.makePlugin()
	}

	if _, ok := m.plugin.(HasConfigStruct); !ok {
		// If plugin doesn't implement HasConfigStruct then we're decoding
		// into an empty PluginConfig object.
		if err := toml.PrimitiveDecode(m.tomlSection, m.configStruct); err != nil {
			return fmt.Errorf("can't decode config for '%s': %s ", m.name, err.Error())
		}
		m.configPrepped = true
		return nil
	}

	// Use reflection to extract the fields (or TOML tag names, if available)
	// of the values that Heka has already extracted so we know they're not
	// required to be specified in the config struct.
	hekaParams := make(map[string]interface{})
	commons := []interface{}{m.commonConfig, m.commonTypedConfig}
	for _, common := range commons {
		if common == nil {
			continue
		}
		rt := reflect.ValueOf(common).Type()
		for i := 0; i < rt.NumField(); i++ {
			sft := rt.Field(i)
			kname := sft.Tag.Get("toml")
			if len(kname) == 0 {
				kname = sft.Name
			}
			hekaParams[kname] = true
		}
	}

	// Finally decode the TOML into the struct. Use of PrimitiveDecodeStrict
	// means that an error will be raised for any config options in the TOML
	// that don't have corresponding attributes on the struct, delta the
	// hekaParams that can be safely excluded.
	err := toml.PrimitiveDecodeStrict(m.tomlSection, m.configStruct, hekaParams)
	if err != nil {
		matches := unknownOptionRegex.FindStringSubmatch(err.Error())
		if len(matches) == 2 {
			// We've got an unrecognized config option.
			return fmt.Errorf("unknown config setting for '%s': %s", m.name, matches[1])
		}
		return err
	}

	m.configPrepped = true
	return nil
}
Esempio n. 3
0
File: config.go Progetto: Jimdo/heka
func LoadHekadConfig(configPath string) (config *HekadConfig, err error) {
	idle, _ := time.ParseDuration("2m")

	config = &HekadConfig{Maxprocs: 1,
		PoolSize:              100,
		DecoderPoolSize:       4,
		ChanSize:              50,
		CpuProfName:           "",
		MemProfName:           "",
		MaxMsgLoops:           4,
		MaxMsgProcessInject:   1,
		MaxMsgProcessDuration: 100000,
		MaxMsgTimerInject:     10,
		MaxPackIdle:           idle,
		BaseDir:               filepath.FromSlash("/var/cache/hekad"),
		ShareDir:              filepath.FromSlash("/usr/share/heka"),
	}

	var configFile map[string]toml.Primitive
	p, err := os.Open(configPath)
	if err != nil {
		return nil, fmt.Errorf("Error opening config file: %s", err)
	}
	fi, err := p.Stat()
	if err != nil {
		return nil, fmt.Errorf("Error fetching config file info: %s", err)
	}

	if fi.IsDir() {
		files, _ := ioutil.ReadDir(configPath)
		for _, f := range files {
			fName := f.Name()
			if strings.HasPrefix(fName, ".") || strings.HasSuffix(fName, ".bak") ||
				strings.HasSuffix(fName, ".tmp") || strings.HasSuffix(fName, "~") {
				// Skip obviously non-relevant files.
				continue
			}
			fPath := filepath.Join(configPath, fName)
			if _, err = toml.DecodeFile(fPath, &configFile); err != nil {
				return nil, fmt.Errorf("Error decoding config file: %s", err)
			}
		}
	} else {
		if _, err = toml.DecodeFile(configPath, &configFile); err != nil {
			return nil, fmt.Errorf("Error decoding config file: %s", err)
		}
	}

	empty_ignore := map[string]interface{}{}
	parsed_config, ok := configFile["hekad"]
	if ok {
		if err = toml.PrimitiveDecodeStrict(parsed_config, config, empty_ignore); err != nil {
			err = fmt.Errorf("Can't unmarshal config: %s", err)
		}
	}

	return
}
Esempio n. 4
0
// If `configable` supports the `HasConfigStruct` interface this will use said
// interface to fetch a config struct object and populate it w/ the values in
// provided `config`. If not, simply returns `config` unchanged.
func LoadConfigStruct(config toml.Primitive, configable interface{}) (
	configStruct interface{}, err error) {

	// On two lines for scoping reasons.
	hasConfigStruct, ok := configable.(HasConfigStruct)
	if !ok {
		// If we don't have a config struct, change it to a PluginConfig
		configStruct = PluginConfig{}
		if err = toml.PrimitiveDecode(config, configStruct); err != nil {
			configStruct = nil
		}
		return
	}

	defer func() {
		// Slight protection against ConfigStruct call into plugin code.
		if r := recover(); r != nil {
			configStruct = nil
			err = fmt.Errorf("ConfigStruct() panicked: %s", r)
		}
	}()

	configStruct = hasConfigStruct.ConfigStruct()

	// Heka defines some common parameters
	// that are defined in the PluginGlobals struct.
	// Use reflection to extract the PluginGlobals fields or TOML tag
	// name if available
	heka_params := make(map[string]interface{})
	pg := PluginGlobals{}
	rt := reflect.ValueOf(pg).Type()
	for i := 0; i < rt.NumField(); i++ {
		sft := rt.Field(i)
		kname := sft.Tag.Get("toml")
		if len(kname) == 0 {
			kname = sft.Name
		}
		heka_params[kname] = true
	}

	if err = toml.PrimitiveDecodeStrict(config, configStruct,
		heka_params); err != nil {
		configStruct = nil
		matches := unknownOptionRegex.FindStringSubmatch(err.Error())
		if len(matches) == 2 {
			// We've got an unrecognized config option.
			err = fmt.Errorf("Unknown config setting: %s", matches[1])
		}
	}
	return
}
Esempio n. 5
0
func LoadConfigStruct(sectionName string, env envconf.Environment,
	config toml.Primitive, configable HasConfigStruct) (
	configStruct interface{}, err error) {

	if configStruct = configable.ConfigStruct(); configStruct == nil {
		return configStruct, nil
	}

	// Global section options
	// SimplePush defines some common parameters
	// that are defined in the ExtensibleGlobals struct.
	// Use reflection to extract the tagged ExtensibleGlobals fields.
	var pg ExtensibleGlobals
	rt := reflect.TypeOf(pg)

	ignoreConfig := make(map[string]interface{}, rt.NumField())
	ignoreEnv := make(map[string]interface{}, rt.NumField())

	for i := 0; i < rt.NumField(); i++ {
		sft := rt.Field(i)
		kname := sft.Tag.Get("toml")
		if len(kname) == 0 {
			kname = sft.Name
		}
		ignoreConfig[kname] = true
		if kname = sft.Tag.Get("env"); len(kname) == 0 {
			kname = sft.Name
		}
		ignoreEnv[toEnvName(sectionName, kname)] = true
	}

	if err = toml.PrimitiveDecodeStrict(config, configStruct, ignoreConfig); err != nil {
		matches := unknownOptionRegex.FindStringSubmatch(err.Error())
		if len(matches) == 2 {
			// We've got an unrecognized config option.
			err = fmt.Errorf("Unknown config setting for section '%s': %s",
				sectionName, matches[1])
		}
		return nil, err
	}

	if err = env.DecodeStrict(toEnvName(sectionName), EnvSep, configStruct, ignoreEnv); err != nil {
		return nil, fmt.Errorf("Invalid environment variable for section '%s': %s",
			sectionName, err)
	}

	return configStruct, nil
}
Esempio n. 6
0
// OrigPrepConfig is the default implementation of the `PrepConfig` method.
func (m *pluginMaker) OrigPrepConfig() (interface{}, error) {
	config := m.Config()
	var err error

	if _, ok := config.(PluginConfig); ok {
		if err = toml.PrimitiveDecode(m.tomlSection, config); err != nil {
			return nil, fmt.Errorf("can't decode config for '%s': %s", m.name, err.Error())
		}
		return config, nil
	}

	// Use reflection to extract the fields (or TOML tag names, if available)
	// of the values that Heka has already extracted so we know they're not
	// required to be specified in the config struct.
	hekaParams := make(map[string]interface{})
	commonTypedConfig, _ := m.OrigPrepCommonTypedConfig()
	commons := []interface{}{m.commonConfig, commonTypedConfig}
	for _, common := range commons {
		if common == nil {
			continue
		}
		rt := reflect.ValueOf(common).Type()
		for i := 0; i < rt.NumField(); i++ {
			sft := rt.Field(i)
			kname := sft.Tag.Get("toml")
			if len(kname) == 0 {
				kname = sft.Name
			}
			hekaParams[kname] = true
		}
	}

	// Finally decode the TOML into the struct. Use of PrimitiveDecodeStrict
	// means that an error will be raised for any config options in the TOML
	// that don't have corresponding attributes on the struct, delta the
	// hekaParams that can be safely excluded.
	err = toml.PrimitiveDecodeStrict(m.tomlSection, config, hekaParams)
	if err != nil {
		matches := unknownOptionRegex.FindStringSubmatch(err.Error())
		if len(matches) == 2 {
			// We've got an unrecognized config option.
			err = fmt.Errorf("unknown config setting for '%s': %s", m.name, matches[1])
		}
		return nil, err
	}

	return config, nil
}
Esempio n. 7
0
func (pdi *ProcessDirectoryInput) loadProcessFile(path string) (config *ProcessInputConfig,
	err error) {

	unparsedConfig := make(map[string]toml.Primitive)
	if _, err = toml.DecodeFile(path, &unparsedConfig); err != nil {
		return
	}
	section, ok := unparsedConfig["ProcessInput"]
	if !ok {
		err = errors.New("No `ProcessInput` section.")
		return
	}
	config = &ProcessInputConfig{
		ParserType:  "token",
		ParseStdout: true,
		Trim:        true,
	}
	if err = toml.PrimitiveDecodeStrict(section, config, nil); err != nil {
		return nil, err
	}
	return
}
Esempio n. 8
0
func LoadHekadConfig(configPath string) (config *HekadConfig, err error) {
	idle, _ := time.ParseDuration("2m")

	config = &HekadConfig{Maxprocs: 1,
		PoolSize:              100,
		ChanSize:              50,
		CpuProfName:           "",
		MemProfName:           "",
		MaxMsgLoops:           4,
		MaxMsgProcessInject:   1,
		MaxMsgProcessDuration: 100000,
		MaxMsgTimerInject:     10,
		MaxPackIdle:           idle,
		BaseDir:               filepath.FromSlash("/var/cache/hekad"),
		ShareDir:              filepath.FromSlash("/usr/share/heka"),
		SampleDenominator:     1000,
		PidFile:               "",
	}

	var configFile map[string]toml.Primitive
	p, err := os.Open(configPath)
	if err != nil {
		return nil, fmt.Errorf("Error opening config file: %s", err)
	}
	fi, err := p.Stat()
	if err != nil {
		return nil, fmt.Errorf("Error fetching config file info: %s", err)
	}

	if fi.IsDir() {
		files, _ := ioutil.ReadDir(configPath)
		for _, f := range files {
			fName := f.Name()
			if !strings.HasSuffix(fName, ".toml") {
				// Skip non *.toml files in a config dir.
				continue
			}
			fPath := filepath.Join(configPath, fName)
			contents, err := pipeline.ReplaceEnvsFile(fPath)
			if err != nil {
				return nil, err
			}
			if _, err = toml.Decode(contents, &configFile); err != nil {
				return nil, fmt.Errorf("Error decoding config file: %s", err)
			}
		}
	} else {
		contents, err := pipeline.ReplaceEnvsFile(configPath)
		if err != nil {
			return nil, err
		}
		if _, err = toml.Decode(contents, &configFile); err != nil {
			return nil, fmt.Errorf("Error decoding config file: %s", err)
		}
	}

	empty_ignore := map[string]interface{}{}
	parsed_config, ok := configFile[pipeline.HEKA_DAEMON]
	if ok {
		if err = toml.PrimitiveDecodeStrict(parsed_config, config, empty_ignore); err != nil {
			err = fmt.Errorf("Can't unmarshal config: %s", err)
		}
	}

	return
}
Esempio n. 9
0
func LoadConfig(configPath *string) *GlobalConfig {
	f, err := os.Open(*configPath)
	if err != nil {
		log.Fatalln("open config file or dir failed: ", *configPath)
	}
	defer f.Close()

	fs, err := os.Stat(*configPath)
	if err != nil {
		log.Fatalln("get stat of file or dir failed: ", *configPath)
	}

	if fs.IsDir() {
		log.Fatalln("config file must be file: ", *configPath)
	}

	if !strings.HasSuffix(*configPath, ".toml") {
		log.Fatalln("config file must has .toml suffix")
	}

	data, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatalln("ioutil.ReadAll config file failed: ", *configPath)
	}

	// log.Println(data)

	globals := &GlobalConfig{}
	_, err = toml.Decode(string(data), &cfgs)
	if err != nil {
		log.Fatalln("toml.Decode data failed: ", err)
	}
	log.Println(cfgs)
	// log.Println(cfgs)

	// kafkacfg := &KafkaInputConfig{}
	base := &BaseConfig{}
	kafkacfg := NewKafkaInputConfig()
	influxcfg := NewInfluxDBOutputConfig()

	empty_ignore := map[string]interface{}{}

	parsed_globals, ok := cfgs["global"]

	if !ok {
		log.Fatalln("global base toml must be set")
	}
	if err = toml.PrimitiveDecodeStrict(parsed_globals, base, empty_ignore); err != nil {
		log.Fatalln("global base decode failed: ", err)
	}

	globals.Base = base

	log.Println(globals.Base)

	parsed_config, ok := cfgs[globals.Base.Input]
	if ok {
		if err = toml.PrimitiveDecodeStrict(parsed_config, kafkacfg, empty_ignore); err != nil {
			// err = fmt.Errorf("Can't unmarshal config: %s", err)
			log.Fatalln("can't unmarshal config: ", err)
		}
		// log.Println(kafkacfg)
		globals.KafkaConfig = kafkacfg
	}
	parsed_config_influxdb, ok := cfgs[globals.Base.Output]
	if ok {
		if err = toml.PrimitiveDecodeStrict(parsed_config_influxdb, influxcfg, empty_ignore); err != nil {
			// err = fmt.Errorf("Can't unmarshal config: %s", err)
			log.Fatalln("can't unmarshal config: ", err)
		}
		// log.Println(kafkacfg)
		globals.InfluxDBConfig = influxcfg
	}

	return globals
}