// OrigPrepCommonTypedConfig is the default implementation of the
// `PrepCommonTypedConfig` function.
func (m *pluginMaker) OrigPrepCommonTypedConfig() (interface{}, error) {
var (
commonTypedConfig interface{}
err error
)
switch m.category {
case "Input":
commonInput := CommonInputConfig{
Retries: getDefaultRetryOptions(),
}
err = toml.PrimitiveDecode(m.tomlSection, &commonInput)
commonTypedConfig = commonInput
case "Filter", "Output":
commonFO := CommonFOConfig{
Retries: getDefaultRetryOptions(),
}
err = toml.PrimitiveDecode(m.tomlSection, &commonFO)
commonTypedConfig = commonFO
case "Splitter":
commonSplitter := CommonSplitterConfig{}
err = toml.PrimitiveDecode(m.tomlSection, &commonSplitter)
commonTypedConfig = commonSplitter
}
if err != nil {
return nil, err
}
return commonTypedConfig, nil
}
// If `configable` supports the `HasConfigStruct` interface this will use said
// interface to fetch a config struct object and populate it w/ the values in
// provided `config`. If not, simply returns `config` unchanged.
func LoadConfigStruct(config toml.Primitive, configable interface{}) (
configStruct interface{}, err error) {
// On two lines for scoping reasons.
hasConfigStruct, ok := configable.(HasConfigStruct)
if !ok {
// If we don't have a config struct, change it to a PluginConfig
configStruct = PluginConfig{}
if err = toml.PrimitiveDecode(config, configStruct); err != nil {
configStruct = nil
}
return
}
defer func() {
// Slight protection against ConfigStruct call into plugin code.
if r := recover(); r != nil {
configStruct = nil
err = fmt.Errorf("ConfigStruct() panicked: %s", r)
}
}()
configStruct = hasConfigStruct.ConfigStruct()
if err = toml.PrimitiveDecode(config, configStruct); err != nil {
configStruct = nil
err = fmt.Errorf("Can't unmarshal config: %s", err)
}
return
}
// NewPluginMaker creates and returns a PluginMaker that can generate running
// plugins for the provided TOML configuration. It will load the plugin type
// and extract any of the Heka-defined common config for the plugin before
// returning.
func NewPluginMaker(name string, pConfig *PipelineConfig, tomlSection toml.Primitive) (
PluginMaker, error) {
// Create the maker, extract plugin type, and make sure the plugin type
// exists.
maker := &pluginMaker{
name: name,
tomlSection: tomlSection,
commonConfig: CommonConfig{},
pConfig: pConfig,
}
var err error
if err = toml.PrimitiveDecode(tomlSection, &maker.commonConfig); err != nil {
return nil, fmt.Errorf("can't decode common config for '%s': %s", name, err)
}
if maker.commonConfig.Typ == "" {
maker.commonConfig.Typ = name
}
constructor, ok := AvailablePlugins[maker.commonConfig.Typ]
if !ok {
return nil, fmt.Errorf("No registered plugin type: %s", maker.commonConfig.Typ)
}
maker.constructor = constructor
// Extract plugin category and any category-specific common (i.e. Heka
// defined) configuration.
maker.category = getPluginCategory(maker.commonConfig.Typ)
if maker.category == "" {
return nil, errors.New("Unrecognized plugin category")
}
switch maker.category {
case "Input":
commonInput := CommonInputConfig{
Retries: getDefaultRetryOptions(),
}
err = toml.PrimitiveDecode(tomlSection, &commonInput)
maker.commonTypedConfig = commonInput
case "Filter", "Output":
commonFO := CommonFOConfig{
Retries: getDefaultRetryOptions(),
}
err = toml.PrimitiveDecode(tomlSection, &commonFO)
maker.commonTypedConfig = commonFO
case "Splitter":
commonSplitter := CommonSplitterConfig{}
err = toml.PrimitiveDecode(tomlSection, &commonSplitter)
maker.commonTypedConfig = commonSplitter
}
if err != nil {
return nil, fmt.Errorf("can't decode common %s config for '%s': %s",
strings.ToLower(maker.category), name, err)
}
return maker, nil
}
func (this *iRunner) Start(conf toml.Primitive) {
plugCommon := &PluginCommonConfig{
Type: "",
Tag: "",
}
if err := toml.PrimitiveDecode(conf, plugCommon); err != nil {
log.Fatalln("toml struct error")
}
input, ok := input_plugins[plugCommon.Type]
if !ok {
log.Fatalln("unkown type ", plugCommon.Type)
}
in := input()
err := in.(Input).Init(plugCommon, conf)
if err != nil {
log.Fatalln("in.(Input).Init", err)
}
err = in.(Input).Run(this)
if err != nil {
log.Fatalln("in.(Input).Run", err)
}
}
// Loads the config for a section supplied, configures the supplied object, and initializes
func LoadConfigForSection(app *Application, sectionName string, obj HasConfigStruct,
env envconf.Environment, configFile ConfigFile) (err error) {
conf, ok := configFile[sectionName]
if !ok {
return fmt.Errorf("Error loading config file, section: %s", sectionName)
}
confStruct := obj.ConfigStruct()
if confStruct == nil {
return nil
}
if err = toml.PrimitiveDecode(conf, confStruct); err != nil {
return fmt.Errorf("Unable to decode config for section '%s': %s",
sectionName, err)
}
if err = env.Decode(toEnvName(sectionName), EnvSep, confStruct); err != nil {
return fmt.Errorf("Invalid environment variable for section '%s': %s",
sectionName, err)
}
err = obj.Init(app, confStruct)
return
}
func (self *TcpInput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) {
log.Println("TcpInput Init")
self.common = pcf
self.config = &TcpInputConfig{
Net: "tcp",
}
if err := toml.PrimitiveDecode(conf, self.config); err != nil {
return fmt.Errorf("Can't unmarshal TcpInput config: %s", err)
}
address, err := net.ResolveTCPAddr(self.config.Net, self.config.Address)
if err != nil {
return fmt.Errorf("ResolveTCPAddress failed: %s\n", err.Error())
}
self.listener, err = net.ListenTCP(self.config.Net, address)
if err != nil {
return fmt.Errorf("ListenTCP failed: %s\n", err.Error())
}
// We're already listening, make sure we clean up if init fails later on.
closeIt := true
defer func() {
if closeIt {
self.listener.Close()
}
}()
if self.config.KeepAlivePeriod != 0 {
self.keepAliveDuration = time.Duration(self.config.KeepAlivePeriod) * time.Second
}
self.stopChan = make(chan bool)
closeIt = false
return nil
}
// This function extracts the Heka specified config options (the poorly named
// "PluginGlobals") from a given plugin's config section. If successful, the
// PluginGlobals are stored on the provided ConfigSection. If unsuccessful,
// ConfigSection will be unchanged and an error will be returned.
func (self *PipelineConfig) loadPluginGlobals(section *ConfigSection) (err error) {
// Set up default retry policy.
pGlobals := new(PluginGlobals)
pGlobals.Retries = RetryOptions{
MaxDelay: "30s",
Delay: "250ms",
MaxRetries: -1,
}
if err = toml.PrimitiveDecode(section.tomlSection, pGlobals); err != nil {
err = fmt.Errorf("Unable to decode config for plugin '%s': %s",
section.name, err)
return
}
if pGlobals.Typ == "" {
pGlobals.Typ = section.name
}
if _, ok := AvailablePlugins[pGlobals.Typ]; !ok {
err = fmt.Errorf("No registered plugin type: %s", pGlobals.Typ)
} else {
section.globals = pGlobals
}
return
}
func main() {
configFile := flag.String("config", "logstreamer.toml", "Heka Logstreamer configuration file")
flag.Parse()
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
fconfig := make(FileConfig)
if _, err := toml.DecodeFile(*configFile, &fconfig); err != nil {
log.Printf("Error decoding config file: %s", err)
return
}
// Filter out logstream inputs
inputs := make(map[string]toml.Primitive)
for name, prim := range fconfig {
basic := new(Basic)
if name == "LogstreamerInput" {
inputs[name] = prim
} else if err := toml.PrimitiveDecode(prim, &basic); err == nil {
if basic.PluginType == "LogstreamerInput" {
inputs[name] = prim
}
}
}
// Go through the logstreams and parse their configs
for name, prim := range inputs {
parseConfig(name, prim)
}
}
func (this *oRunner) Start(cf toml.Primitive) {
plugCommon := &PluginCommonConfig{
Type: "",
Tag: "",
Decoder: "",
Encoder: "",
}
log.Printf("cf %+v", cf)
if err := toml.PrimitiveDecode(cf, plugCommon); err != nil {
log.Fatalln("toml struct error")
}
output_plugin, ok := output_plugins[plugCommon.Type]
if !ok {
log.Fatalln("unkown type ", plugCommon.Type)
}
out := output_plugin()
err := out.(Output).Init(plugCommon, cf)
if err != nil {
log.Fatalln("out.(Output).Init", err)
}
err = out.(Output).Run(this)
if err != nil {
log.Fatalln("out.(Output).Run", err)
}
}
func (this *TailsInput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) {
this.common = pcf
this.config = &TailsInputConfig{
LogDirectory: "/var/log",
JournalDirectory: "/tmp/",
//FileMatch: "*.log",
RescanInterval: "1m",
SyncInterval: 2,
}
this.files = make([]string, 0)
if err := toml.PrimitiveDecode(conf, this.config); err != nil {
return fmt.Errorf("Can't unmarshal tails config: %s", err)
}
if this.config.FileMatch == "" {
return errors.New("`file_match` setting is required.")
}
if len(this.config.FileMatch) > 0 && this.config.FileMatch[len(this.config.FileMatch)-1:] != "$" {
this.config.FileMatch += "$"
}
// Setup the rescan interval.
if this.rescanInterval, err = time.ParseDuration(this.config.RescanInterval); err != nil {
return
}
if !fileExists(this.config.JournalDirectory) {
if err = os.MkdirAll(filepath.Dir(this.config.JournalDirectory), 0766); err != nil {
return
}
}
return nil
}
// Creates a FilterRunner for the specified sandbox name and configuration
func createRunner(dir, name string, configSection toml.Primitive) (FilterRunner, error) {
var err error
var pluginGlobals PluginGlobals
wrapper := new(PluginWrapper)
wrapper.name = name
pluginGlobals.Retries = RetryOptions{
MaxDelay: "30s",
Delay: "250ms",
MaxRetries: -1,
}
if err = toml.PrimitiveDecode(configSection, &pluginGlobals); err != nil {
return nil, fmt.Errorf("Unable to decode config for plugin: %s, error: %s",
wrapper.name, err.Error())
}
if pluginGlobals.Typ != "SandboxFilter" {
return nil, fmt.Errorf("Plugin must be a SandboxFilter, received %s",
pluginGlobals.Typ)
}
// Create plugin, test config object generation.
wrapper.pluginCreator, _ = AvailablePlugins[pluginGlobals.Typ]
plugin := wrapper.pluginCreator()
var config interface{}
if config, err = LoadConfigStruct(configSection, plugin); err != nil {
return nil, fmt.Errorf("Can't load config for '%s': %s", wrapper.name, err)
}
wrapper.configCreator = func() interface{} { return config }
conf := config.(*sandbox.SandboxConfig)
conf.ScriptFilename = filepath.Join(dir, fmt.Sprintf("%s.%s", wrapper.name, conf.ScriptType))
if wantsName, ok := plugin.(WantsName); ok {
wantsName.SetName(wrapper.name)
}
// Apply configuration to instantiated plugin.
if err = plugin.(Plugin).Init(config); err != nil {
return nil, fmt.Errorf("Initialization failed for '%s': %s", name, err)
}
runner := NewFORunner(wrapper.name, plugin.(Plugin), &pluginGlobals)
runner.name = wrapper.name
if pluginGlobals.Ticker != 0 {
runner.tickLength = time.Duration(pluginGlobals.Ticker) * time.Second
}
var matcher *MatchRunner
if pluginGlobals.Matcher != "" {
if matcher, err = NewMatchRunner(pluginGlobals.Matcher,
pluginGlobals.Signer, runner); err != nil {
return nil, fmt.Errorf("Can't create message matcher for '%s': %s",
wrapper.name, err)
}
runner.matcher = matcher
}
return runner, nil
}
func (this *Pipeline) LoadConfig(plugConfig map[string]toml.Primitive) error {
for k, v := range plugConfig {
log.Printf("v %+v", v)
plugCommon := &PluginCommonConfig{}
if err := toml.PrimitiveDecode(v, plugCommon); err != nil {
return fmt.Errorf("Can't unmarshal config: %s", err)
}
pluginType := getPluginType(plugCommon.Type)
if pluginType == "" {
continue
}
if plugCommon.Tag == "" {
log.Println("Tag empty")
}
switch pluginType {
case "Input":
this.InputRunners = append(this.InputRunners, v)
case "Output":
this.OutputRunners = append(this.OutputRunners, v)
case "Encoder":
this.EncodeRunners = append(this.EncodeRunners, v)
case "Decoder":
this.DecodeRunners = append(this.DecodeRunners, v)
}
log.Printf("%s => %s", k, plugCommon.Type)
}
return nil
}
// Load an extensible section that has a type keyword
func LoadExtensibleSection(app *Application, sectionName string,
extensions AvailableExtensions, env envconf.Environment,
configFile ConfigFile) (obj HasConfigStruct, err error) {
confSection := new(ExtensibleGlobals)
conf, ok := configFile[sectionName]
if !ok {
return nil, fmt.Errorf("Missing section '%s'", sectionName)
}
if err = toml.PrimitiveDecode(conf, confSection); err != nil {
return nil, err
}
if err = env.Decode(toEnvName(sectionName), EnvSep, confSection); err != nil {
return nil, err
}
ext, ok := extensions.Get(confSection.Typ)
if !ok {
//TODO: Add log info to indicate using "default"
return nil, fmt.Errorf("No type '%s' available to load for section '%s'",
confSection.Typ, sectionName)
}
obj = ext()
loadedConfig, err := LoadConfigStruct(sectionName, env, conf, obj)
if err != nil {
return nil, err
}
err = obj.Init(app, loadedConfig)
return obj, err
}
func (self *UdpInput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) {
log.Println("UdpInput Init")
self.common = pcf
self.config = &UdpInputConfig{
Net: "udp4",
}
if err := toml.PrimitiveDecode(conf, self.config); err != nil {
return fmt.Errorf("Can't unmarshal UdpInput config: %s", err)
}
address, err := net.ResolveUDPAddr(self.config.Net, self.config.Address)
if err != nil {
return fmt.Errorf("ResolveUDPAddress failed: %s\n", err.Error())
}
self.listener, err = net.ListenUDP(self.config.Net, address)
if err != nil {
return fmt.Errorf("ListenUDP failed: %s\n", err.Error())
}
self.listener.SetReadBuffer(1048576)
closeIt := true
defer func() {
if closeIt {
self.listener.Close()
}
}()
self.stopChan = make(chan bool)
self.inChan = make(chan UdpPack, 1024)
closeIt = false
return nil
}
// loadSection must be passed a plugin name and the config for that plugin. It
// will create a PluginWrapper (i.e. a factory).
func (md *MultiDecoder) loadSection(sectionName string,
configSection toml.Primitive) (plugin Decoder, err error) {
var ok bool
var pluginGlobals PluginGlobals
var pluginType string
wrapper := new(PluginWrapper)
wrapper.Name = sectionName
// Setup default retry policy
pluginGlobals.Retries = RetryOptions{
MaxDelay: "30s",
Delay: "250ms",
MaxRetries: -1,
}
if err = toml.PrimitiveDecode(configSection, &pluginGlobals); err != nil {
err = fmt.Errorf("%s Unable to decode config for plugin: %s, error: %s", md.Name, wrapper.Name, err.Error())
md.log(err.Error())
return
}
if pluginGlobals.Typ == "" {
pluginType = sectionName
} else {
pluginType = pluginGlobals.Typ
}
if wrapper.PluginCreator, ok = AvailablePlugins[pluginType]; !ok {
err = fmt.Errorf("%s No such plugin: %s (type: %s)", md.Name, wrapper.Name, pluginType)
md.log(err.Error())
return
}
// Create plugin, test config object generation.
plugin = wrapper.PluginCreator().(Decoder)
var config interface{}
if config, err = LoadConfigStruct(configSection, plugin); err != nil {
err = fmt.Errorf("%s Can't load config for %s '%s': %s", md.Name,
sectionName,
wrapper.Name, err)
md.log(err.Error())
return
}
wrapper.ConfigCreator = func() interface{} { return config }
if wantsName, ok := plugin.(WantsName); ok {
wantsName.SetName(wrapper.Name)
}
// Apply configuration to instantiated plugin.
if err = plugin.(Plugin).Init(config); err != nil {
err = fmt.Errorf("Initialization failed for '%s': %s",
sectionName, err)
md.log(err.Error())
return
}
return
}
// PrepConfig generates a config struct for the plugin (instantiating an
// instance of the plugin to do so, if necessary) and decodes the TOML config
// into the generated struct.
func (m *pluginMaker) PrepConfig() error {
if m.configPrepped {
// Already done, just return.
return nil
}
if m.configStruct == nil {
m.makeConfig()
} else if m.plugin == nil {
m.plugin = m.makePlugin()
}
if _, ok := m.plugin.(HasConfigStruct); !ok {
// If plugin doesn't implement HasConfigStruct then we're decoding
// into an empty PluginConfig object.
if err := toml.PrimitiveDecode(m.tomlSection, m.configStruct); err != nil {
return fmt.Errorf("can't decode config for '%s': %s ", m.name, err.Error())
}
m.configPrepped = true
return nil
}
// Use reflection to extract the fields (or TOML tag names, if available)
// of the values that Heka has already extracted so we know they're not
// required to be specified in the config struct.
hekaParams := make(map[string]interface{})
commons := []interface{}{m.commonConfig, m.commonTypedConfig}
for _, common := range commons {
if common == nil {
continue
}
rt := reflect.ValueOf(common).Type()
for i := 0; i < rt.NumField(); i++ {
sft := rt.Field(i)
kname := sft.Tag.Get("toml")
if len(kname) == 0 {
kname = sft.Name
}
hekaParams[kname] = true
}
}
// Finally decode the TOML into the struct. Use of PrimitiveDecodeStrict
// means that an error will be raised for any config options in the TOML
// that don't have corresponding attributes on the struct, delta the
// hekaParams that can be safely excluded.
err := toml.PrimitiveDecodeStrict(m.tomlSection, m.configStruct, hekaParams)
if err != nil {
matches := unknownOptionRegex.FindStringSubmatch(err.Error())
if len(matches) == 2 {
// We've got an unrecognized config option.
return fmt.Errorf("unknown config setting for '%s': %s", m.name, matches[1])
}
return err
}
m.configPrepped = true
return nil
}
func main() {
configFile := flag.String("config", "logstreamer.toml", "Heka Logstreamer configuration file")
flag.Parse()
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
p, err := os.Open(*configFile)
if err != nil {
client.LogError.Fatalf("Error opening config file: %s", err)
}
fi, err := p.Stat()
if err != nil {
client.LogError.Fatalf("Error fetching config file info: %s", err)
}
fconfig := make(FileConfig)
if fi.IsDir() {
files, _ := ioutil.ReadDir(*configFile)
for _, f := range files {
fName := f.Name()
if strings.HasPrefix(fName, ".") || strings.HasSuffix(fName, ".bak") ||
strings.HasSuffix(fName, ".tmp") || strings.HasSuffix(fName, "~") {
// Skip obviously non-relevant files.
continue
}
fPath := filepath.Join(*configFile, fName)
if _, err = toml.DecodeFile(fPath, &fconfig); err != nil {
client.LogError.Fatalf("Error decoding config file: %s", err)
}
}
} else {
if _, err := toml.DecodeFile(*configFile, &fconfig); err != nil {
client.LogError.Fatalf("Error decoding config file: %s", err)
}
}
// Filter out logstream inputs
inputs := make(map[string]toml.Primitive)
for name, prim := range fconfig {
basic := new(Basic)
if name == "LogstreamerInput" {
inputs[name] = prim
} else if err := toml.PrimitiveDecode(prim, &basic); err == nil {
if basic.PluginType == "LogstreamerInput" {
inputs[name] = prim
}
}
}
// Go through the logstreams and parse their configs
for name, prim := range inputs {
parseConfig(name, prim)
}
}
func (self *UdpOutput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) error {
var err error
log.Println("UdpOutput Init.")
self.config = &UdpOutputConfig{
Net: "udp",
// Defines maximum size of udp data for IPv4.
MaxMessageSize: 65507,
}
if err := toml.PrimitiveDecode(conf, self.config); err != nil {
return fmt.Errorf("Can't unmarshal UdpOutput config: %s", err)
}
if self.config.MaxMessageSize < 512 {
return fmt.Errorf("Maximum message size can't be smaller than 512 bytes.")
}
if self.config.Net == "unixgram" {
if runtime.GOOS == "windows" {
return errors.New("Can't use Unix datagram sockets on Windows.")
}
var unixAddr, lAddr *net.UnixAddr
unixAddr, err = net.ResolveUnixAddr(self.config.Net, self.config.Address)
if err != nil {
return fmt.Errorf("Error resolving unixgram address '%s': %s", self.config.Address,
err.Error())
}
if self.config.LocalAddress != "" {
lAddr, err = net.ResolveUnixAddr(self.config.Net, self.config.LocalAddress)
if err != nil {
return fmt.Errorf("Error resolving local unixgram address '%s': %s",
self.config.LocalAddress, err.Error())
}
}
if self.conn, err = net.DialUnix(self.config.Net, lAddr, unixAddr); err != nil {
return fmt.Errorf("Can't connect to '%s': %s", self.config.Address,
err.Error())
}
} else {
var udpAddr, lAddr *net.UDPAddr
if udpAddr, err = net.ResolveUDPAddr(self.config.Net, self.config.Address); err != nil {
return fmt.Errorf("Error resolving UDP address '%s': %s", self.config.Address,
err.Error())
}
if self.config.LocalAddress != "" {
lAddr, err = net.ResolveUDPAddr(self.config.Net, self.config.LocalAddress)
if err != nil {
return fmt.Errorf("Error resolving local UDP address '%s': %s",
self.config.Address, err.Error())
}
}
if self.conn, err = net.DialUDP(self.config.Net, lAddr, udpAddr); err != nil {
return fmt.Errorf("Can't connect to '%s': %s", self.config.Address,
err.Error())
}
}
return err
}
func (self *MongodbOutput) Init(conf toml.Primitive) error {
log.Println("MongodbOutput Init.")
self.config = &MongodbOutputConfig{
Host: "localhost",
Port: "27017",
Database: "test",
Collection: "test",
}
if err := toml.PrimitiveDecode(conf, self.config); err != nil {
return fmt.Errorf("Can't unmarshal MongodbOutput config: %s", err)
}
return nil
}
// If `configable` supports the `HasConfigStruct` interface this will use said
// interface to fetch a config struct object and populate it w/ the values in
// provided `config`. If not, simply returns `config` unchanged.
func LoadConfigStruct(config toml.Primitive, configable interface{}) (
configStruct interface{}, err error) {
// On two lines for scoping reasons.
hasConfigStruct, ok := configable.(HasConfigStruct)
if !ok {
// If we don't have a config struct, change it to a PluginConfig
configStruct = PluginConfig{}
if err = toml.PrimitiveDecode(config, configStruct); err != nil {
configStruct = nil
}
return
}
defer func() {
// Slight protection against ConfigStruct call into plugin code.
if r := recover(); r != nil {
configStruct = nil
err = fmt.Errorf("ConfigStruct() panicked: %s", r)
}
}()
configStruct = hasConfigStruct.ConfigStruct()
// Heka defines some common parameters
// that are defined in the PluginGlobals struct.
// Use reflection to extract the PluginGlobals fields or TOML tag
// name if available
heka_params := make(map[string]interface{})
pg := PluginGlobals{}
rt := reflect.ValueOf(pg).Type()
for i := 0; i < rt.NumField(); i++ {
sft := rt.Field(i)
kname := sft.Tag.Get("toml")
if len(kname) == 0 {
kname = sft.Name
}
heka_params[kname] = true
}
if err = toml.PrimitiveDecodeStrict(config, configStruct,
heka_params); err != nil {
configStruct = nil
matches := unknownOptionRegex.FindStringSubmatch(err.Error())
if len(matches) == 2 {
// We've got an unrecognized config option.
err = fmt.Errorf("Unknown config setting: %s", matches[1])
}
}
return
}
// Parses a Heka message and extracts the information necessary to start a new
// SandboxFilter
func (this *SandboxManagerFilter) loadSandbox(fr pipeline.FilterRunner,
h pipeline.PluginHelper, dir string, msg *message.Message) (err error) {
fv, _ := msg.GetFieldValue("config")
if config, ok := fv.(string); ok {
var configFile pipeline.ConfigFile
if _, err = toml.Decode(config, &configFile); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
} else {
for name, conf := range configFile {
name = getSandboxName(fr.Name(), name)
if _, ok := h.Filter(name); ok {
// todo support reload
return fmt.Errorf("loadSandbox failed: %s is already running", name)
}
fr.LogMessage(fmt.Sprintf("Loading: %s", name))
confFile := filepath.Join(dir, fmt.Sprintf("%s.toml", name))
err = ioutil.WriteFile(confFile, []byte(config), 0600)
if err != nil {
return
}
var sbc SandboxConfig
if err = toml.PrimitiveDecode(conf, &sbc); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
}
scriptFile := filepath.Join(dir, fmt.Sprintf("%s.%s", name, sbc.ScriptType))
err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
// check/clear the old state preservation file
// this avoids issues with changes to the data model since the last load
// and prevents holes in the graph from looking like anomalies
os.Remove(filepath.Join(pipeline.PrependBaseDir(DATA_DIR), name+DATA_EXT))
var runner pipeline.FilterRunner
runner, err = this.createRunner(dir, name, conf)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
err = h.PipelineConfig().AddFilterRunner(runner)
if err == nil {
this.currentFilters++
}
break // only interested in the first item
}
}
}
return
}
func (self *FileOutput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) error {
var err error
var intPerm int64
log.Println("FileOutput Init.")
self.common = pcf
self.config = &FileOutputConfig{
Perm: "644",
RotationInterval: 0,
FlushInterval: 1000,
FolderPerm: "700",
}
if err := toml.PrimitiveDecode(conf, self.config); err != nil {
return fmt.Errorf("Can't unmarshal FileOutput config: %s", err)
}
if intPerm, err = strconv.ParseInt(self.config.FolderPerm, 8, 32); err != nil {
err = fmt.Errorf("FileOutput '%s' can't parse `folder_perm`, is it an octal integer string?",
self.config.Path)
return err
}
self.folderPerm = os.FileMode(intPerm)
if intPerm, err = strconv.ParseInt(self.config.Perm, 8, 32); err != nil {
err = fmt.Errorf("FileOutput '%s' can't parse `perm`, is it an octal integer string?",
self.config.Path)
return err
}
self.perm = os.FileMode(intPerm)
self.closing = make(chan struct{})
switch self.config.RotationInterval {
case 0:
// date rotation is disabled
self.path = self.config.Path
case 1, 4, 12, 24:
// RotationInterval value is allowed
self.startRotateNotifier()
default:
err = fmt.Errorf("Parameter 'rotation_interval' must be one of: 0, 1, 4, 12, 24.")
return err
}
if err = self.openFile(); err != nil {
err = fmt.Errorf("FileOutput '%s' error opening file: %s", self.path, err)
close(self.closing)
return err
}
self.batchChan = make(chan *outBatch)
self.backChan = make(chan *outBatch, 2) // Never block on the hand-back
self.rotateChan = make(chan time.Time)
return err
}
// Parses a Heka message and extracts the information necessary to start a new
// SandboxFilter
func (this *SandboxManagerFilter) loadSandbox(fr pipeline.FilterRunner,
h pipeline.PluginHelper, dir string, msg *message.Message) (err error) {
fv, _ := msg.GetFieldValue("config")
if config, ok := fv.(string); ok {
var configFile pipeline.ConfigFile
if _, err = toml.Decode(config, &configFile); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
}
for name, conf := range configFile {
name = getSandboxName(fr.Name(), name)
if _, ok := h.Filter(name); ok {
// todo support reload
return fmt.Errorf("loadSandbox failed: %s is already running", name)
}
fr.LogMessage(fmt.Sprintf("Loading: %s", name))
confFile := filepath.Join(dir, fmt.Sprintf("%s.toml", name))
err = ioutil.WriteFile(confFile, []byte(config), 0600)
if err != nil {
return
}
var sbc SandboxConfig
// Default, will get overwritten if necessary
sbc.ScriptType = "lua"
if err = toml.PrimitiveDecode(conf, &sbc); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
}
scriptFile := filepath.Join(dir, fmt.Sprintf("%s.%s", name, sbc.ScriptType))
err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
var runner pipeline.FilterRunner
runner, err = this.createRunner(dir, name, conf)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
err = this.pConfig.AddFilterRunner(runner)
if err == nil {
atomic.AddInt32(&this.currentFilters, 1)
}
break // only interested in the first item
}
}
return
}
//http://play.golang.org/p/fOWJXgcfKO
func (this *RegexDecoder) Init(conf toml.Primitive) (err error) {
this.config = &RegexDecoderConfig{}
if err = toml.PrimitiveDecode(conf, this.config); err != nil {
return fmt.Errorf("Can't unmarshal regexdecoder config: %s", err)
}
if this.Match, err = regexp.Compile(this.config.MatchRegex); err != nil {
err = fmt.Errorf("RegexDecoder: %s", err)
return
}
if this.Match.NumSubexp() == 0 {
err = fmt.Errorf("RegexDecoder regex must contain capture groups")
return
}
return nil
}
// OrigPrepConfig is the default implementation of the `PrepConfig` method.
func (m *pluginMaker) OrigPrepConfig() (interface{}, error) {
config := m.Config()
var err error
if _, ok := config.(PluginConfig); ok {
if err = toml.PrimitiveDecode(m.tomlSection, config); err != nil {
return nil, fmt.Errorf("can't decode config for '%s': %s", m.name, err.Error())
}
return config, nil
}
// Use reflection to extract the fields (or TOML tag names, if available)
// of the values that Heka has already extracted so we know they're not
// required to be specified in the config struct.
hekaParams := make(map[string]interface{})
commonTypedConfig, _ := m.OrigPrepCommonTypedConfig()
commons := []interface{}{m.commonConfig, commonTypedConfig}
for _, common := range commons {
if common == nil {
continue
}
rt := reflect.ValueOf(common).Type()
for i := 0; i < rt.NumField(); i++ {
sft := rt.Field(i)
kname := sft.Tag.Get("toml")
if len(kname) == 0 {
kname = sft.Name
}
hekaParams[kname] = true
}
}
// Finally decode the TOML into the struct. Use of PrimitiveDecodeStrict
// means that an error will be raised for any config options in the TOML
// that don't have corresponding attributes on the struct, delta the
// hekaParams that can be safely excluded.
err = toml.PrimitiveDecodeStrict(m.tomlSection, config, hekaParams)
if err != nil {
matches := unknownOptionRegex.FindStringSubmatch(err.Error())
if len(matches) == 2 {
// We've got an unrecognized config option.
err = fmt.Errorf("unknown config setting for '%s': %s", m.name, matches[1])
}
return nil, err
}
return config, nil
}
func (this *SandboxManagerFilter) loadSandbox(fr FilterRunner,
h PluginHelper, dir string, msg *message.Message) (err error) {
fv, _ := msg.GetFieldValue("config")
if config, ok := fv.(string); ok {
var configFile ConfigFile
if _, err = toml.Decode(config, &configFile); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
} else {
for name, conf := range configFile {
name = getSandboxName(fr.Name(), name)
if _, ok := h.Filter(name); ok {
// todo support reload
return fmt.Errorf("loadSandbox failed: %s is already running", name)
}
fr.LogMessage(fmt.Sprintf("Loading: %s", name))
confFile := path.Join(dir, fmt.Sprintf("%s.toml", name))
err = ioutil.WriteFile(confFile, []byte(config), 0600)
if err != nil {
return
}
var sbfc SandboxFilterConfig
if err = toml.PrimitiveDecode(conf, &sbfc); err != nil {
return fmt.Errorf("loadSandbox failed: %s\n", err)
}
scriptFile := path.Join(dir, fmt.Sprintf("%s.%s", name, sbfc.Sbc.ScriptType))
err = ioutil.WriteFile(scriptFile, []byte(msg.GetPayload()), 0600)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
var runner FilterRunner
runner, err = createRunner(dir, name, conf)
if err != nil {
removeAll(dir, fmt.Sprintf("%s.*", name))
return
}
err = h.PipelineConfig().AddFilterRunner(runner)
if err == nil {
this.currentFilters++
}
break // only interested in the first item
}
}
}
return
}
// NewPluginMaker creates and returns a PluginMaker that can generate running
// plugins for the provided TOML configuration. It will load the plugin type
// and extract any of the Heka-defined common config for the plugin before
// returning.
func NewPluginMaker(name string, pConfig *PipelineConfig, tomlSection toml.Primitive) (
PluginMaker, error) {
// Create the maker, extract plugin type, and make sure the plugin type
// exists.
maker := &pluginMaker{
name: name,
tomlSection: tomlSection,
commonConfig: CommonConfig{},
pConfig: pConfig,
}
var err error
if err = toml.PrimitiveDecode(tomlSection, &maker.commonConfig); err != nil {
return nil, fmt.Errorf("can't decode common config for '%s': %s", name, err)
}
if maker.commonConfig.Typ == "" {
maker.commonConfig.Typ = name
}
constructor, ok := AvailablePlugins[maker.commonConfig.Typ]
if !ok {
return nil, fmt.Errorf("No registered plugin type: %s", maker.commonConfig.Typ)
}
maker.constructor = constructor
maker.plugin = maker.makePlugin() // Only used to generate config structs.
// Extract plugin category and any category-specific common (i.e. Heka
// defined) configuration.
maker.category = getPluginCategory(maker.commonConfig.Typ)
if maker.category == "" {
return nil, errors.New("Unrecognized plugin category")
}
maker.prepCommonTypedConfig = maker.OrigPrepCommonTypedConfig
_, err = maker.prepCommonTypedConfig()
if err != nil {
return nil, fmt.Errorf("can't decode common %s config for '%s': %s",
strings.ToLower(maker.category), name, err)
}
maker.prepConfig = maker.OrigPrepConfig
return maker, nil
}
// Loads the config for a section supplied, configures the supplied object, and initializes
func LoadConfigForSection(app *Application, sectionName string, obj HasConfigStruct,
env envconf.Environment, configFile ConfigFile) (err error) {
conf, ok := configFile[sectionName]
if !ok {
return fmt.Errorf("Error loading config file, section: %s", sectionName)
}
confStruct := obj.ConfigStruct()
if confStruct == nil {
return nil
}
if err = toml.PrimitiveDecode(conf, confStruct); err != nil {
return fmt.Errorf("Unable to decode config for section '%s': %s",
sectionName, err)
}
return LoadConfigFromEnvironment(app, sectionName, obj, env, confStruct)
}
func parseConfig(name string, prim toml.Primitive) {
config := LogstreamerConfig{
OldestDuration: "720h",
Differentiator: []string{name},
LogDirectory: "/var/log",
}
if err := toml.PrimitiveDecode(prim, &config); err != nil {
client.LogError.Printf("Error decoding config file: %s", err)
return
}
if len(config.FileMatch) > 0 && config.FileMatch[len(config.FileMatch)-1:] != "$" {
config.FileMatch += "$"
}
sp := &logstreamer.SortPattern{
FileMatch: config.FileMatch,
Translation: config.Translation,
Priority: config.Priority,
Differentiator: config.Differentiator,
}
oldest, _ := time.ParseDuration(config.OldestDuration)
ls, err := logstreamer.NewLogstreamSet(sp, oldest, config.LogDirectory, "")
if err != nil {
client.LogError.Fatalf("Error initializing LogstreamSet: %s\n", err.Error())
}
streams, errs := ls.ScanForLogstreams()
if errs.IsError() {
client.LogError.Fatalf("Error scanning: %s\n", errs)
}
fmt.Printf("Found %d Logstream(s) for section [%s].\n", len(streams), name)
for _, name := range streams {
stream, _ := ls.GetLogstream(name)
fmt.Printf("\nLogstream name: [%s]\n", name)
fmt.Printf("Files: %d (printing oldest to newest)\n", len(stream.GetLogfiles()))
for _, logfile := range stream.GetLogfiles() {
fmt.Printf("\t%s\n", logfile.FileName)
}
}
}
func (self *KafkaInput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) {
log.Println("KafkaInput Init.")
self.common = pcf
hn, err := os.Hostname()
if err != nil {
hn = "kamanclient"
}
self.config = &KafkaInputConfig{
ClientId: hn,
Partitions: 0,
FlushInterval: 1000,
}
if err = toml.PrimitiveDecode(conf, self.config); err != nil {
return fmt.Errorf("Can't unmarshal KafkaInput config: %s", err)
}
if len(self.config.Addrs) == 0 {
return errors.New("addrs must have at least one entry")
}
if len(self.config.Topic) == 0 {
return fmt.Errorf("topic is empty")
}
bcf := kafka.NewBrokerConf(self.config.ClientId)
bcf.AllowTopicCreation = false
//bcf.Logger = &stdLogger{}
self.broker, err = kafka.Dial(self.config.Addrs, bcf)
if err != nil {
return fmt.Errorf("cannot connect to kafka cluster: %s", err)
}
defer self.broker.Close()
consumerconf := kafka.NewConsumerConf(self.config.Topic, self.config.Partition)
self.consumer, err = self.broker.Consumer(consumerconf)
if err != nil {
return fmt.Errorf("cannot create kafka consumer for %s:%d: %s", self.config.Topic, self.config.Partition, err)
}
return err
}