func LoadHekadConfig(filename string) (config *HekadConfig, err error) {
config = &HekadConfig{Maxprocs: 1,
PoolSize: 100,
DecoderPoolSize: 4,
ChanSize: 50,
CpuProfName: "",
MemProfName: "",
MaxMsgLoops: 4,
MaxMsgProcessInject: 1,
MaxMsgTimerInject: 10,
}
var configFile map[string]toml.Primitive
if _, err = toml.DecodeFile(filename, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
empty_ignore := map[string]interface{}{}
parsed_config, ok := configFile["hekad"]
if ok {
if err = toml.PrimitiveDecodeStrict(parsed_config, &config, empty_ignore); err != nil {
err = fmt.Errorf("Can't unmarshal config: %s", err)
}
}
return
}
// PrepConfig generates a config struct for the plugin (instantiating an
// instance of the plugin to do so, if necessary) and decodes the TOML config
// into the generated struct.
func (m *pluginMaker) PrepConfig() error {
if m.configPrepped {
// Already done, just return.
return nil
}
if m.configStruct == nil {
m.makeConfig()
} else if m.plugin == nil {
m.plugin = m.makePlugin()
}
if _, ok := m.plugin.(HasConfigStruct); !ok {
// If plugin doesn't implement HasConfigStruct then we're decoding
// into an empty PluginConfig object.
if err := toml.PrimitiveDecode(m.tomlSection, m.configStruct); err != nil {
return fmt.Errorf("can't decode config for '%s': %s ", m.name, err.Error())
}
m.configPrepped = true
return nil
}
// Use reflection to extract the fields (or TOML tag names, if available)
// of the values that Heka has already extracted so we know they're not
// required to be specified in the config struct.
hekaParams := make(map[string]interface{})
commons := []interface{}{m.commonConfig, m.commonTypedConfig}
for _, common := range commons {
if common == nil {
continue
}
rt := reflect.ValueOf(common).Type()
for i := 0; i < rt.NumField(); i++ {
sft := rt.Field(i)
kname := sft.Tag.Get("toml")
if len(kname) == 0 {
kname = sft.Name
}
hekaParams[kname] = true
}
}
// Finally decode the TOML into the struct. Use of PrimitiveDecodeStrict
// means that an error will be raised for any config options in the TOML
// that don't have corresponding attributes on the struct, delta the
// hekaParams that can be safely excluded.
err := toml.PrimitiveDecodeStrict(m.tomlSection, m.configStruct, hekaParams)
if err != nil {
matches := unknownOptionRegex.FindStringSubmatch(err.Error())
if len(matches) == 2 {
// We've got an unrecognized config option.
return fmt.Errorf("unknown config setting for '%s': %s", m.name, matches[1])
}
return err
}
m.configPrepped = true
return nil
}
func LoadHekadConfig(configPath string) (config *HekadConfig, err error) {
idle, _ := time.ParseDuration("2m")
config = &HekadConfig{Maxprocs: 1,
PoolSize: 100,
DecoderPoolSize: 4,
ChanSize: 50,
CpuProfName: "",
MemProfName: "",
MaxMsgLoops: 4,
MaxMsgProcessInject: 1,
MaxMsgProcessDuration: 100000,
MaxMsgTimerInject: 10,
MaxPackIdle: idle,
BaseDir: filepath.FromSlash("/var/cache/hekad"),
ShareDir: filepath.FromSlash("/usr/share/heka"),
}
var configFile map[string]toml.Primitive
p, err := os.Open(configPath)
if err != nil {
return nil, fmt.Errorf("Error opening config file: %s", err)
}
fi, err := p.Stat()
if err != nil {
return nil, fmt.Errorf("Error fetching config file info: %s", err)
}
if fi.IsDir() {
files, _ := ioutil.ReadDir(configPath)
for _, f := range files {
fName := f.Name()
if strings.HasPrefix(fName, ".") || strings.HasSuffix(fName, ".bak") ||
strings.HasSuffix(fName, ".tmp") || strings.HasSuffix(fName, "~") {
// Skip obviously non-relevant files.
continue
}
fPath := filepath.Join(configPath, fName)
if _, err = toml.DecodeFile(fPath, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
}
} else {
if _, err = toml.DecodeFile(configPath, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
}
empty_ignore := map[string]interface{}{}
parsed_config, ok := configFile["hekad"]
if ok {
if err = toml.PrimitiveDecodeStrict(parsed_config, config, empty_ignore); err != nil {
err = fmt.Errorf("Can't unmarshal config: %s", err)
}
}
return
}
// If `configable` supports the `HasConfigStruct` interface this will use said
// interface to fetch a config struct object and populate it w/ the values in
// provided `config`. If not, simply returns `config` unchanged.
func LoadConfigStruct(config toml.Primitive, configable interface{}) (
configStruct interface{}, err error) {
// On two lines for scoping reasons.
hasConfigStruct, ok := configable.(HasConfigStruct)
if !ok {
// If we don't have a config struct, change it to a PluginConfig
configStruct = PluginConfig{}
if err = toml.PrimitiveDecode(config, configStruct); err != nil {
configStruct = nil
}
return
}
defer func() {
// Slight protection against ConfigStruct call into plugin code.
if r := recover(); r != nil {
configStruct = nil
err = fmt.Errorf("ConfigStruct() panicked: %s", r)
}
}()
configStruct = hasConfigStruct.ConfigStruct()
// Heka defines some common parameters
// that are defined in the PluginGlobals struct.
// Use reflection to extract the PluginGlobals fields or TOML tag
// name if available
heka_params := make(map[string]interface{})
pg := PluginGlobals{}
rt := reflect.ValueOf(pg).Type()
for i := 0; i < rt.NumField(); i++ {
sft := rt.Field(i)
kname := sft.Tag.Get("toml")
if len(kname) == 0 {
kname = sft.Name
}
heka_params[kname] = true
}
if err = toml.PrimitiveDecodeStrict(config, configStruct,
heka_params); err != nil {
configStruct = nil
matches := unknownOptionRegex.FindStringSubmatch(err.Error())
if len(matches) == 2 {
// We've got an unrecognized config option.
err = fmt.Errorf("Unknown config setting: %s", matches[1])
}
}
return
}
func LoadConfigStruct(sectionName string, env envconf.Environment,
config toml.Primitive, configable HasConfigStruct) (
configStruct interface{}, err error) {
if configStruct = configable.ConfigStruct(); configStruct == nil {
return configStruct, nil
}
// Global section options
// SimplePush defines some common parameters
// that are defined in the ExtensibleGlobals struct.
// Use reflection to extract the tagged ExtensibleGlobals fields.
var pg ExtensibleGlobals
rt := reflect.TypeOf(pg)
ignoreConfig := make(map[string]interface{}, rt.NumField())
ignoreEnv := make(map[string]interface{}, rt.NumField())
for i := 0; i < rt.NumField(); i++ {
sft := rt.Field(i)
kname := sft.Tag.Get("toml")
if len(kname) == 0 {
kname = sft.Name
}
ignoreConfig[kname] = true
if kname = sft.Tag.Get("env"); len(kname) == 0 {
kname = sft.Name
}
ignoreEnv[toEnvName(sectionName, kname)] = true
}
if err = toml.PrimitiveDecodeStrict(config, configStruct, ignoreConfig); err != nil {
matches := unknownOptionRegex.FindStringSubmatch(err.Error())
if len(matches) == 2 {
// We've got an unrecognized config option.
err = fmt.Errorf("Unknown config setting for section '%s': %s",
sectionName, matches[1])
}
return nil, err
}
if err = env.DecodeStrict(toEnvName(sectionName), EnvSep, configStruct, ignoreEnv); err != nil {
return nil, fmt.Errorf("Invalid environment variable for section '%s': %s",
sectionName, err)
}
return configStruct, nil
}
// OrigPrepConfig is the default implementation of the `PrepConfig` method.
func (m *pluginMaker) OrigPrepConfig() (interface{}, error) {
config := m.Config()
var err error
if _, ok := config.(PluginConfig); ok {
if err = toml.PrimitiveDecode(m.tomlSection, config); err != nil {
return nil, fmt.Errorf("can't decode config for '%s': %s", m.name, err.Error())
}
return config, nil
}
// Use reflection to extract the fields (or TOML tag names, if available)
// of the values that Heka has already extracted so we know they're not
// required to be specified in the config struct.
hekaParams := make(map[string]interface{})
commonTypedConfig, _ := m.OrigPrepCommonTypedConfig()
commons := []interface{}{m.commonConfig, commonTypedConfig}
for _, common := range commons {
if common == nil {
continue
}
rt := reflect.ValueOf(common).Type()
for i := 0; i < rt.NumField(); i++ {
sft := rt.Field(i)
kname := sft.Tag.Get("toml")
if len(kname) == 0 {
kname = sft.Name
}
hekaParams[kname] = true
}
}
// Finally decode the TOML into the struct. Use of PrimitiveDecodeStrict
// means that an error will be raised for any config options in the TOML
// that don't have corresponding attributes on the struct, delta the
// hekaParams that can be safely excluded.
err = toml.PrimitiveDecodeStrict(m.tomlSection, config, hekaParams)
if err != nil {
matches := unknownOptionRegex.FindStringSubmatch(err.Error())
if len(matches) == 2 {
// We've got an unrecognized config option.
err = fmt.Errorf("unknown config setting for '%s': %s", m.name, matches[1])
}
return nil, err
}
return config, nil
}
func (pdi *ProcessDirectoryInput) loadProcessFile(path string) (config *ProcessInputConfig,
err error) {
unparsedConfig := make(map[string]toml.Primitive)
if _, err = toml.DecodeFile(path, &unparsedConfig); err != nil {
return
}
section, ok := unparsedConfig["ProcessInput"]
if !ok {
err = errors.New("No `ProcessInput` section.")
return
}
config = &ProcessInputConfig{
ParserType: "token",
ParseStdout: true,
Trim: true,
}
if err = toml.PrimitiveDecodeStrict(section, config, nil); err != nil {
return nil, err
}
return
}
func LoadHekadConfig(configPath string) (config *HekadConfig, err error) {
idle, _ := time.ParseDuration("2m")
config = &HekadConfig{Maxprocs: 1,
PoolSize: 100,
ChanSize: 50,
CpuProfName: "",
MemProfName: "",
MaxMsgLoops: 4,
MaxMsgProcessInject: 1,
MaxMsgProcessDuration: 100000,
MaxMsgTimerInject: 10,
MaxPackIdle: idle,
BaseDir: filepath.FromSlash("/var/cache/hekad"),
ShareDir: filepath.FromSlash("/usr/share/heka"),
SampleDenominator: 1000,
PidFile: "",
}
var configFile map[string]toml.Primitive
p, err := os.Open(configPath)
if err != nil {
return nil, fmt.Errorf("Error opening config file: %s", err)
}
fi, err := p.Stat()
if err != nil {
return nil, fmt.Errorf("Error fetching config file info: %s", err)
}
if fi.IsDir() {
files, _ := ioutil.ReadDir(configPath)
for _, f := range files {
fName := f.Name()
if !strings.HasSuffix(fName, ".toml") {
// Skip non *.toml files in a config dir.
continue
}
fPath := filepath.Join(configPath, fName)
contents, err := pipeline.ReplaceEnvsFile(fPath)
if err != nil {
return nil, err
}
if _, err = toml.Decode(contents, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
}
} else {
contents, err := pipeline.ReplaceEnvsFile(configPath)
if err != nil {
return nil, err
}
if _, err = toml.Decode(contents, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
}
empty_ignore := map[string]interface{}{}
parsed_config, ok := configFile[pipeline.HEKA_DAEMON]
if ok {
if err = toml.PrimitiveDecodeStrict(parsed_config, config, empty_ignore); err != nil {
err = fmt.Errorf("Can't unmarshal config: %s", err)
}
}
return
}
func LoadConfig(configPath *string) *GlobalConfig {
f, err := os.Open(*configPath)
if err != nil {
log.Fatalln("open config file or dir failed: ", *configPath)
}
defer f.Close()
fs, err := os.Stat(*configPath)
if err != nil {
log.Fatalln("get stat of file or dir failed: ", *configPath)
}
if fs.IsDir() {
log.Fatalln("config file must be file: ", *configPath)
}
if !strings.HasSuffix(*configPath, ".toml") {
log.Fatalln("config file must has .toml suffix")
}
data, err := ioutil.ReadAll(f)
if err != nil {
log.Fatalln("ioutil.ReadAll config file failed: ", *configPath)
}
// log.Println(data)
globals := &GlobalConfig{}
_, err = toml.Decode(string(data), &cfgs)
if err != nil {
log.Fatalln("toml.Decode data failed: ", err)
}
log.Println(cfgs)
// log.Println(cfgs)
// kafkacfg := &KafkaInputConfig{}
base := &BaseConfig{}
kafkacfg := NewKafkaInputConfig()
influxcfg := NewInfluxDBOutputConfig()
empty_ignore := map[string]interface{}{}
parsed_globals, ok := cfgs["global"]
if !ok {
log.Fatalln("global base toml must be set")
}
if err = toml.PrimitiveDecodeStrict(parsed_globals, base, empty_ignore); err != nil {
log.Fatalln("global base decode failed: ", err)
}
globals.Base = base
log.Println(globals.Base)
parsed_config, ok := cfgs[globals.Base.Input]
if ok {
if err = toml.PrimitiveDecodeStrict(parsed_config, kafkacfg, empty_ignore); err != nil {
// err = fmt.Errorf("Can't unmarshal config: %s", err)
log.Fatalln("can't unmarshal config: ", err)
}
// log.Println(kafkacfg)
globals.KafkaConfig = kafkacfg
}
parsed_config_influxdb, ok := cfgs[globals.Base.Output]
if ok {
if err = toml.PrimitiveDecodeStrict(parsed_config_influxdb, influxcfg, empty_ignore); err != nil {
// err = fmt.Errorf("Can't unmarshal config: %s", err)
log.Fatalln("can't unmarshal config: ", err)
}
// log.Println(kafkacfg)
globals.InfluxDBConfig = influxcfg
}
return globals
}