func LoadHekadConfig(configPath string) (config *HekadConfig, err error) { idle, _ := time.ParseDuration("2m") config = &HekadConfig{Maxprocs: 1, PoolSize: 100, DecoderPoolSize: 4, ChanSize: 50, CpuProfName: "", MemProfName: "", MaxMsgLoops: 4, MaxMsgProcessInject: 1, MaxMsgProcessDuration: 100000, MaxMsgTimerInject: 10, MaxPackIdle: idle, BaseDir: filepath.FromSlash("/var/cache/hekad"), ShareDir: filepath.FromSlash("/usr/share/heka"), } var configFile map[string]toml.Primitive p, err := os.Open(configPath) if err != nil { return nil, fmt.Errorf("Error opening config file: %s", err) } fi, err := p.Stat() if err != nil { return nil, fmt.Errorf("Error fetching config file info: %s", err) } if fi.IsDir() { files, _ := ioutil.ReadDir(configPath) for _, f := range files { fName := f.Name() if strings.HasPrefix(fName, ".") || strings.HasSuffix(fName, ".bak") || strings.HasSuffix(fName, ".tmp") || strings.HasSuffix(fName, "~") { // Skip obviously non-relevant files. continue } fPath := filepath.Join(configPath, fName) if _, err = toml.DecodeFile(fPath, &configFile); err != nil { return nil, fmt.Errorf("Error decoding config file: %s", err) } } } else { if _, err = toml.DecodeFile(configPath, &configFile); err != nil { return nil, fmt.Errorf("Error decoding config file: %s", err) } } empty_ignore := map[string]interface{}{} parsed_config, ok := configFile["hekad"] if ok { if err = toml.PrimitiveDecodeStrict(parsed_config, config, empty_ignore); err != nil { err = fmt.Errorf("Can't unmarshal config: %s", err) } } return }
func main() { configFile := flag.String("config", "logstreamer.toml", "Heka Logstreamer configuration file") flag.Parse() if flag.NFlag() == 0 { flag.PrintDefaults() os.Exit(0) } p, err := os.Open(*configFile) if err != nil { client.LogError.Fatalf("Error opening config file: %s", err) } fi, err := p.Stat() if err != nil { client.LogError.Fatalf("Error fetching config file info: %s", err) } fconfig := make(FileConfig) if fi.IsDir() { files, _ := ioutil.ReadDir(*configFile) for _, f := range files { fName := f.Name() if strings.HasPrefix(fName, ".") || strings.HasSuffix(fName, ".bak") || strings.HasSuffix(fName, ".tmp") || strings.HasSuffix(fName, "~") { // Skip obviously non-relevant files. continue } fPath := filepath.Join(*configFile, fName) if _, err = toml.DecodeFile(fPath, &fconfig); err != nil { client.LogError.Fatalf("Error decoding config file: %s", err) } } } else { if _, err := toml.DecodeFile(*configFile, &fconfig); err != nil { client.LogError.Fatalf("Error decoding config file: %s", err) } } // Filter out logstream inputs inputs := make(map[string]toml.Primitive) for name, prim := range fconfig { basic := new(Basic) if name == "LogstreamerInput" { inputs[name] = prim } else if err := toml.PrimitiveDecode(prim, &basic); err == nil { if basic.PluginType == "LogstreamerInput" { inputs[name] = prim } } } // Go through the logstreams and parse their configs for name, prim := range inputs { parseConfig(name, prim) } }
func main() { configFile := flag.String("config", "logstreamer.toml", "Heka Logstreamer configuration file") flag.Parse() if flag.NFlag() == 0 { flag.PrintDefaults() os.Exit(0) } fconfig := make(FileConfig) if _, err := toml.DecodeFile(*configFile, &fconfig); err != nil { log.Printf("Error decoding config file: %s", err) return } // Filter out logstream inputs inputs := make(map[string]toml.Primitive) for name, prim := range fconfig { basic := new(Basic) if name == "LogstreamerInput" { inputs[name] = prim } else if err := toml.PrimitiveDecode(prim, &basic); err == nil { if basic.PluginType == "LogstreamerInput" { inputs[name] = prim } } } // Go through the logstreams and parse their configs for name, prim := range inputs { parseConfig(name, prim) } }
func (pdi *ProcessDirectoryInput) loadProcessFile(path string) (*ProcessEntry, error) { var err error unparsedConfig := make(map[string]toml.Primitive) if _, err = toml.DecodeFile(path, &unparsedConfig); err != nil { return nil, err } section, ok := unparsedConfig["ProcessInput"] if !ok { err = errors.New("No `ProcessInput` section.") return nil, err } maker, err := NewPluginMaker("ProcessInput", pdi.pConfig, section) if err != nil { return nil, fmt.Errorf("can't create plugin maker: %s", err) } mutMaker := maker.(MutableMaker) mutMaker.SetName(path) if err = mutMaker.PrepConfig(); err != nil { return nil, fmt.Errorf("can't prep config: %s", err) } entry := &ProcessEntry{ maker: mutMaker, config: mutMaker.Config().(*ProcessInputConfig), } return entry, nil }
// On Heka restarts this function reloads all previously running SandboxFilters // using the script, configuration, and preservation files in the working // directory. func (this *SandboxManagerFilter) restoreSandboxes(fr pipeline.FilterRunner, h pipeline.PluginHelper, dir string) { glob := fmt.Sprintf("%s-*.toml", getNormalizedName(fr.Name())) if matches, err := filepath.Glob(filepath.Join(dir, glob)); err == nil { for _, fn := range matches { var configFile pipeline.ConfigFile if _, err = toml.DecodeFile(fn, &configFile); err != nil { fr.LogError(fmt.Errorf("restoreSandboxes failed: %s\n", err)) continue } else { for _, conf := range configFile { var runner pipeline.FilterRunner name := path.Base(fn[:len(fn)-5]) fr.LogMessage(fmt.Sprintf("Loading: %s", name)) runner, err = this.createRunner(dir, name, conf) if err != nil { fr.LogError(fmt.Errorf("createRunner failed: %s\n", err.Error())) removeAll(dir, fmt.Sprintf("%s.*", name)) break } err = h.PipelineConfig().AddFilterRunner(runner) if err != nil { fr.LogError(err) } else { atomic.AddInt32(&this.currentFilters, 1) } break // only interested in the first item } } } } }
// LoadFromConfigFile loads a TOML configuration file and stores the // result in the value pointed to by config. The maps in the config // will be initialized as needed. // // The PipelineConfig should be already initialized before passed in via // its Init function. func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) { var configFile ConfigFile if _, err = toml.DecodeFile(filename, &configFile); err != nil { return fmt.Errorf("Error decoding config file: %s", err) } // Load all the plugins var errcnt uint for name, conf := range configFile { if name == "hekad" { continue } log.Printf("Loading: [%s]\n", name) errcnt += self.loadSection(name, conf) } // Add JSON/PROTOCOL_BUFFER decoders if none were configured var configDefault ConfigFile toml.Decode(defaultDecoderTOML, &configDefault) dWrappers := self.DecoderWrappers if _, ok := dWrappers["ProtobufDecoder"]; !ok { log.Println("Loading: [ProtobufDecoder]") errcnt += self.loadSection("ProtobufDecoder", configDefault["ProtobufDecoder"]) } if errcnt != 0 { return fmt.Errorf("%d errors loading plugins", errcnt) } return }
func LoadHekadConfig(filename string) (config *HekadConfig, err error) { config = &HekadConfig{Maxprocs: 1, PoolSize: 100, DecoderPoolSize: 4, ChanSize: 50, CpuProfName: "", MemProfName: "", MaxMsgLoops: 4, MaxMsgProcessInject: 1, MaxMsgTimerInject: 10, } var configFile map[string]toml.Primitive if _, err = toml.DecodeFile(filename, &configFile); err != nil { return nil, fmt.Errorf("Error decoding config file: %s", err) } empty_ignore := map[string]interface{}{} parsed_config, ok := configFile["hekad"] if ok { if err = toml.PrimitiveDecodeStrict(parsed_config, &config, empty_ignore); err != nil { err = fmt.Errorf("Can't unmarshal config: %s", err) } } return }
func main() { configFile := flag.String("config", "sbmgr.toml", "Sandbox manager configuration file") scriptFile := flag.String("script", "xyz.lua", "Sandbox script file") scriptConfig := flag.String("scriptconfig", "xyz.toml", "Sandbox script configuration file") filterName := flag.String("filtername", "filter", "Sandbox filter name (used on unload)") action := flag.String("action", "load", "Sandbox manager action") flag.Parse() var config SbmgrConfig if _, err := toml.DecodeFile(*configFile, &config); err != nil { log.Printf("Error decoding config file: %s", err) return } sender, err := client.NewNetworkSender("tcp", config.IpAddress) if err != nil { log.Fatalf("Error creating sender: %s\n", err.Error()) } encoder := client.NewProtobufEncoder(&config.Signer) manager := client.NewClient(sender, encoder) hostname, _ := os.Hostname() msg := &message.Message{} msg.SetType("heka.control.sandbox") msg.SetTimestamp(time.Now().UnixNano()) msg.SetUuid(uuid.NewRandom()) msg.SetHostname(hostname) switch *action { case "load": code, err := ioutil.ReadFile(*scriptFile) if err != nil { log.Printf("Error reading scriptFile: %s\n", err.Error()) return } msg.SetPayload(string(code)) conf, err := ioutil.ReadFile(*scriptConfig) if err != nil { log.Printf("Error reading scriptConfig: %s\n", err.Error()) return } f, _ := message.NewField("config", string(conf), "toml") msg.AddField(f) case "unload": f, _ := message.NewField("name", *filterName, "") msg.AddField(f) default: log.Printf("Invalid action: %s", *action) } f1, _ := message.NewField("action", *action, "") msg.AddField(f1) err = manager.SendMessage(msg) if err != nil { log.Printf("Error sending message: %s\n", err.Error()) } }
// Handles reading a TOML based configuration file, and loading an // initialized Application, ready to Run func LoadApplicationFromFileName(filename string, logging int) ( app *Application, err error) { var configFile ConfigFile if _, err = toml.DecodeFile(filename, &configFile); err != nil { return nil, fmt.Errorf("Error decoding config file: %s", err) } env := envconf.Load() return LoadApplication(configFile, env, logging) }
func main() { var config Config config_file := flag.String("config", filepath.FromSlash("/etc/heka-server.toml"), "Config file for log server") if _, err := toml.DecodeFile(*config_file, &config); err != nil { fmt.Println(err) return } fmt.Println(config) }
// LoadHeimdallConfig accepts the path of a configuration file // to load. Additionally, it configures defaults for values that // are not specified within the configuration passed. func LoadHeimdallConfig(path string) (config *Config, err error) { config = &Config{CleanupInterval: "3m", Port: ":9001", Cert: "cert.pem", Key: "key.pem", Auth: AuthConfig{DBPath: "/var/cache/heimdall/auth"}, Blacklist: BlacklistConfig{DBPath: "/var/cache/heimdall/blacklist", Path: "/blacklist/", DefaultTTL: "1h"}, Whitelist: WhitelistConfig{DBPath: "/var/cache/heimdall/whitelist", Path: "/whitelist/", DefaultTTL: "72h"}, PanOS: PanOSConfig{Enabled: false, Path: "/panos/"}} if _, err := toml.DecodeFile(path, config); err != nil { return nil, fmt.Errorf("Problem decoding configuration file: %s", err) } return config, nil }
// LoadFromConfigFile loads a TOML configuration file and stores the // result in the value pointed to by config. The maps in the config // will be initialized as needed. // // The PipelineConfig should be already initialized before passed in via // its Init function. func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) { var configFile ConfigFile if _, err = toml.DecodeFile(filename, &configFile); err != nil { return fmt.Errorf("Error decoding config file: %s", err) } // Load all the plugins var errcnt uint for name, conf := range configFile { log.Println("Loading: ", name) errcnt += self.loadSection(name, conf) } // Add JSON/PROTOCOL_BUFFER decoders if none were configured var configDefault ConfigFile toml.Decode(defaultDecoderTOML, &configDefault) dWrappers := self.DecoderWrappers if _, ok := dWrappers["JsonDecoder"]; !ok { log.Println("Loading: JsonDecoder") errcnt += self.loadSection("JsonDecoder", configDefault["JsonDecoder"]) } if _, ok := dWrappers["ProtobufDecoder"]; !ok { log.Println("Loading: ProtobufDecoder") errcnt += self.loadSection("ProtobufDecoder", configDefault["ProtobufDecoder"]) } // Create / prep the DecoderSet pool var dRunner DecoderRunner for i := 0; i < Globals().DecoderPoolSize; i++ { if self.DecoderSets[i], err = newDecoderSet(dWrappers); err != nil { log.Println(err) errcnt += 1 } for _, dRunner = range self.DecoderSets[i].AllByName() { dRunner.Start(self, &self.decodersWg) } self.decodersChan <- self.DecoderSets[i] } if errcnt != 0 { return fmt.Errorf("%d errors loading plugins", errcnt) } return }
func (this *SandboxManagerFilter) Init(config interface{}) (err error) { conf := config.(*SandboxManagerFilterConfig) this.maxFilters = conf.MaxFilters this.workingDirectory, _ = filepath.Abs(conf.WorkingDirectory) if err = os.MkdirAll(this.workingDirectory, 0700); err != nil { return err } filename := path.Join(this.workingDirectory, "config.toml") _, err = os.Stat(filename) if err == nil { var configFile ConfigFile if _, err := toml.DecodeFile(filename, &configFile); err != nil { return fmt.Errorf("Error decoding config file: %s", err) } } return nil }
func (pdi *ProcessDirectoryInput) loadProcessFile(path string) (*ProcessEntry, error) { var err error unparsedConfig := make(map[string]toml.Primitive) if _, err = toml.DecodeFile(path, &unparsedConfig); err != nil { return nil, err } section, ok := unparsedConfig["ProcessInput"] if !ok { err = errors.New("No `ProcessInput` section.") return nil, err } maker, err := NewPluginMaker("ProcessInput", pdi.pConfig, section) if err != nil { return nil, fmt.Errorf("can't create plugin maker: %s", err) } mutMaker := maker.(MutableMaker) mutMaker.SetName(path) prepCommonTypedConfig := func() (interface{}, error) { commonTypedConfig, err := mutMaker.OrigPrepCommonTypedConfig() if err != nil { return nil, err } commonInput := commonTypedConfig.(CommonInputConfig) commonInput.Retries = RetryOptions{ MaxDelay: "30s", Delay: "250ms", MaxRetries: -1, } if commonInput.CanExit == nil { b := true commonInput.CanExit = &b } return commonInput, nil } mutMaker.SetPrepCommonTypedConfig(prepCommonTypedConfig) entry := &ProcessEntry{ maker: mutMaker, } return entry, nil }
func (pdi *ProcessDirectoryInput) loadProcessFile(path string) (config *ProcessInputConfig, err error) { unparsedConfig := make(map[string]toml.Primitive) if _, err = toml.DecodeFile(path, &unparsedConfig); err != nil { return } section, ok := unparsedConfig["ProcessInput"] if !ok { err = errors.New("No `ProcessInput` section.") return } config = &ProcessInputConfig{ ParserType: "token", ParseStdout: true, Trim: true, } if err = toml.PrimitiveDecodeStrict(section, config, nil); err != nil { return nil, err } return }
func main() { configFile := flag.String("config", "sbmgr.toml", "Sandbox manager configuration file") scriptFile := flag.String("script", "xyz.lua", "Sandbox script file") scriptConfig := flag.String("scriptconfig", "xyz.toml", "Sandbox script configuration file") filterName := flag.String("filtername", "filter", "Sandbox filter name (used on unload)") action := flag.String("action", "load", "Sandbox manager action") flag.Parse() var config SbmgrConfig if _, err := toml.DecodeFile(*configFile, &config); err != nil { client.LogError.Printf("Error decoding config file: %s", err) return } var sender *client.NetworkSender var err error if config.UseTls { var goTlsConfig *tls.Config goTlsConfig, err = tcp.CreateGoTlsConfig(&config.Tls) if err != nil { client.LogError.Fatalf("Error creating TLS config: %s\n", err) } sender, err = client.NewTlsSender("tcp", config.IpAddress, goTlsConfig) } else { sender, err = client.NewNetworkSender("tcp", config.IpAddress) } if err != nil { client.LogError.Fatalf("Error creating sender: %s\n", err.Error()) } encoder := client.NewProtobufEncoder(&config.Signer) manager := client.NewClient(sender, encoder) hostname, _ := os.Hostname() msg := &message.Message{} msg.SetLogger(pipeline.HEKA_DAEMON) // identify the message as 'internal' for filtering purposes msg.SetType("heka.control.sandbox") msg.SetTimestamp(time.Now().UnixNano()) msg.SetUuid(uuid.NewRandom()) msg.SetHostname(hostname) switch *action { case "load": code, err := ioutil.ReadFile(*scriptFile) if err != nil { client.LogError.Printf("Error reading scriptFile: %s\n", err.Error()) return } msg.SetPayload(string(code)) conf, err := ioutil.ReadFile(*scriptConfig) if err != nil { client.LogError.Printf("Error reading scriptConfig: %s\n", err.Error()) return } f, _ := message.NewField("config", string(conf), "toml") msg.AddField(f) case "unload": f, _ := message.NewField("name", *filterName, "") msg.AddField(f) default: client.LogError.Printf("Invalid action: %s", *action) } f1, _ := message.NewField("action", *action, "") msg.AddField(f1) err = manager.SendMessage(msg) if err != nil { client.LogError.Printf("Error sending message: %s\n", err.Error()) } }
// Loads all plugin configuration from a TOML configuration file. The // PipelineConfig should be already initialized via the Init function before // this method is called. func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) { var configFile ConfigFile if _, err = toml.DecodeFile(filename, &configFile); err != nil { return fmt.Errorf("Error decoding config file: %s", err) } var ( errcnt uint protobufDRegistered bool protobufERegistered bool ) sectionsByCategory := make(map[string][]*ConfigSection) // Load all the plugin globals and file them by category. for name, conf := range configFile { if name == HEKA_DAEMON { continue } log.Printf("Pre-loading: [%s]\n", name) section := &ConfigSection{ name: name, tomlSection: conf, } if err = self.loadPluginGlobals(section); err != nil { self.log(err.Error()) errcnt++ continue } category := getPluginCategory(section.globals.Typ) if category == "" { self.log(fmt.Sprintf("Type doesn't contain valid plugin name: %s\n", section.globals.Typ)) errcnt++ continue } section.category = category if section.globals.Typ == "MultiDecoder" { // Special case MultiDecoders so we can make sure they get // registered *after* all possible subdecoders. sectionsByCategory["MultiDecoder"] = append(sectionsByCategory["MultiDecoder"], section) } else { sectionsByCategory[category] = append(sectionsByCategory[category], section) } if name == "ProtobufDecoder" { protobufDRegistered = true } if name == "ProtobufEncoder" { protobufERegistered = true } } // Make sure ProtobufDecoder is registered. if !protobufDRegistered { var configDefault ConfigFile toml.Decode(protobufDecoderToml, &configDefault) log.Println("Pre-loading: [ProtobufDecoder]") section := &ConfigSection{ name: "ProtobufDecoder", category: "Decoder", tomlSection: configDefault["ProtobufDecoder"], } if err = self.loadPluginGlobals(section); err != nil { // This really shouldn't happen. self.log(err.Error()) errcnt++ } else { sectionsByCategory["Decoder"] = append(sectionsByCategory["Decoder"], section) } } // Make sure ProtobufEncoder is registered. if !protobufERegistered { var configDefault ConfigFile toml.Decode(protobufEncoderToml, &configDefault) log.Println("Pre-loading: [ProtobufEncoder]") section := &ConfigSection{ name: "ProtobufEncoder", category: "Encoder", tomlSection: configDefault["ProtobufEncoder"], } if err = self.loadPluginGlobals(section); err != nil { // This really shouldn't happen. self.log(err.Error()) errcnt++ } else { sectionsByCategory["Encoder"] = append(sectionsByCategory["Encoder"], section) } } multiDecoders := make([]multiDecoderNode, len(sectionsByCategory["MultiDecoder"])) multiConfigs := make(map[string]*ConfigSection) for i, section := range sectionsByCategory["MultiDecoder"] { multiConfigs[section.name] = section multiDecoders[i] = newMultiDecoderNode(section.name, subsFromSection(section.tomlSection)) } multiDecoders, err = orderDependencies(multiDecoders) if err != nil { return err } for i, d := range multiDecoders { sectionsByCategory["MultiDecoder"][i] = multiConfigs[d.name] } // Append MultiDecoders to the end of the Decoders list. sectionsByCategory["Decoder"] = append(sectionsByCategory["Decoder"], sectionsByCategory["MultiDecoder"]...) // Force decoders and encoders to be registered before the other plugin // types are initialized so we know they'll be there for inputs and // outputs to use during initialization. order := []string{"Decoder", "Encoder", "Input", "Filter", "Output"} for _, category := range order { for _, section := range sectionsByCategory[category] { log.Printf("Loading: [%s]\n", section.name) if err = self.loadSection(section); err != nil { self.log(err.Error()) errcnt++ } } } if errcnt != 0 { return fmt.Errorf("%d errors loading plugins", errcnt) } return }
func main() { configFile := flag.String("config", "sbmgrload.toml", "Sandbox manager load configuration file") action := flag.String("action", "load", "load/unload") numItems := flag.Int("num", 1, "Number of sandboxes to load/unload") flag.Parse() code := ` lastTime = os.time() * 1e9 lastCount = 0 count = 0 rate = 0.0 rates = {} function process_message () count = count + 1 return 0 end function timer_event(ns) local msgsSent = count - lastCount if msgsSent == 0 then return end local elapsedTime = ns - lastTime if elapsedTime == 0 then return end lastCount = count lastTime = ns rate = msgsSent / (elapsedTime / 1e9) rates[#rates+1] = rate output(string.format("Got %d messages. %0.2f msg/sec", count, rate)) inject_message() local samples = #rates if samples == 10 then -- generate a summary every 10 samples table.sort(rates) local min = rates[1] local max = rates[samples] local sum = 0 for i, val in ipairs(rates) do sum = sum + val end output(string.format("AGG Sum. Min: %0.2f Max: %0.2f Mean: %0.2f", min, max, sum/samples)) inject_message() rates = {} end end ` confFmt := ` [CounterSandbox%d] type = "SandboxFilter" message_matcher = "Type == 'hekabench'" ticker_interval = 1.0 [CounterSandbox%d.settings] type = "lua" filename = "" preserve_data = true memory_limit = 32767 instruction_limit = 1000 output_limit = 1024 ` var config SbmgrConfig if _, err := toml.DecodeFile(*configFile, &config); err != nil { log.Printf("Error decoding config file: %s", err) return } sender, err := client.NewNetworkSender("tcp", config.IpAddress) if err != nil { log.Fatalf("Error creating sender: %s\n", err.Error()) } encoder := client.NewProtobufEncoder(&config.Signer) manager := client.NewClient(sender, encoder) hostname, _ := os.Hostname() switch *action { case "load": for i := 0; i < *numItems; i++ { conf := fmt.Sprintf(confFmt, i, i) msg := &message.Message{} msg.SetType("heka.control.sandbox") msg.SetTimestamp(time.Now().UnixNano()) msg.SetUuid(uuid.NewRandom()) msg.SetHostname(hostname) msg.SetPayload(code) f, _ := message.NewField("config", conf, message.Field_RAW) msg.AddField(f) f1, _ := message.NewField("action", *action, message.Field_RAW) msg.AddField(f1) err = manager.SendMessage(msg) if err != nil { log.Printf("Error sending message: %s\n", err.Error()) } } case "unload": for i := 0; i < *numItems; i++ { msg := &message.Message{} msg.SetType("heka.control.sandbox") msg.SetTimestamp(time.Now().UnixNano()) msg.SetUuid(uuid.NewRandom()) msg.SetHostname(hostname) msg.SetPayload(string(code)) f, _ := message.NewField("name", fmt.Sprintf("CounterSandbox%d", i), message.Field_RAW) msg.AddField(f) f1, _ := message.NewField("action", *action, message.Field_RAW) msg.AddField(f1) err = manager.SendMessage(msg) if err != nil { log.Printf("Error sending message: %s\n", err.Error()) } } default: log.Printf("Invalid action: %s\n", *action) } }
func main() { configFile := flag.String("config", "sbmgrload.toml", "Sandbox manager load configuration file") action := flag.String("action", "load", "load/unload") numItems := flag.Int("num", 1, "Number of sandboxes to load/unload") flag.Parse() code := ` require "string" require "table" require "os" lastTime = os.time() * 1e9 lastCount = 0 count = 0 rate = 0.0 rates = {} function process_message () count = count + 1 return 0 end function timer_event(ns) local msgsSent = count - lastCount if msgsSent == 0 then return end local elapsedTime = ns - lastTime if elapsedTime == 0 then return end lastCount = count lastTime = ns rate = msgsSent / (elapsedTime / 1e9) rates[#rates+1] = rate inject_payload("txt", "rate", string.format("Got %d messages. %0.2f msg/sec", count, rate)) local samples = #rates if samples == 10 then -- generate a summary every 10 samples table.sort(rates) local min = rates[1] local max = rates[samples] local sum = 0 for i, val in ipairs(rates) do sum = sum + val end inject_payload("txt", "summary", string.format("AGG Sum. Min: %0.2f Max: %0.2f Mean: %0.2f", min, max, sum/samples)) rates = {} end end ` confFmt := ` [CounterSandbox%d] type = "SandboxFilter" message_matcher = "Type == 'hekabench'" ticker_interval = 1 filename = "" preserve_data = true ` var config SbmgrConfig if _, err := toml.DecodeFile(*configFile, &config); err != nil { client.LogError.Printf("Error decoding config file: %s", err) return } var sender *client.NetworkSender var err error if config.UseTls { var goTlsConfig *tls.Config goTlsConfig, err = tcp.CreateGoTlsConfig(&config.Tls) if err != nil { client.LogError.Fatalf("Error creating TLS config: %s\n", err) } sender, err = client.NewTlsSender("tcp", config.IpAddress, goTlsConfig) } else { sender, err = client.NewNetworkSender("tcp", config.IpAddress) } if err != nil { client.LogError.Fatalf("Error creating sender: %s\n", err.Error()) } encoder := client.NewProtobufEncoder(&config.Signer) manager := client.NewClient(sender, encoder) hostname, _ := os.Hostname() switch *action { case "load": for i := 0; i < *numItems; i++ { conf := fmt.Sprintf(confFmt, i) msg := &message.Message{} msg.SetLogger(pipeline.HEKA_DAEMON) msg.SetType("heka.control.sandbox") msg.SetTimestamp(time.Now().UnixNano()) msg.SetUuid(uuid.NewRandom()) msg.SetHostname(hostname) msg.SetPayload(code) f, _ := message.NewField("config", conf, "toml") msg.AddField(f) f1, _ := message.NewField("action", *action, "") msg.AddField(f1) err = manager.SendMessage(msg) if err != nil { client.LogError.Printf("Error sending message: %s\n", err.Error()) } } case "unload": for i := 0; i < *numItems; i++ { msg := &message.Message{} msg.SetLogger(pipeline.HEKA_DAEMON) msg.SetType("heka.control.sandbox") msg.SetTimestamp(time.Now().UnixNano()) msg.SetUuid(uuid.NewRandom()) msg.SetHostname(hostname) f, _ := message.NewField("name", fmt.Sprintf("CounterSandbox%d", i), "") msg.AddField(f) f1, _ := message.NewField("action", *action, "") msg.AddField(f1) err = manager.SendMessage(msg) if err != nil { client.LogError.Printf("Error sending message: %s\n", err.Error()) } } default: client.LogError.Printf("Invalid action: %s\n", *action) } }
func main() { configFile := flag.String("config", "flood.toml", "Heka Flood configuration file") configTest := flag.String("test", "default", "Test section to load") flag.Parse() if flag.NFlag() == 0 { flag.PrintDefaults() os.Exit(0) } var config FloodConfig if _, err := toml.DecodeFile(*configFile, &config); err != nil { client.LogError.Printf("Error decoding config file: %s", err) return } var test FloodTest var ok bool if test, ok = config[*configTest]; !ok { client.LogError.Printf("Configuration test: '%s' was not found", *configTest) return } if test.MsgInterval != "" { var err error if test.msgInterval, err = time.ParseDuration(test.MsgInterval); err != nil { client.LogError.Printf("Invalid message_interval duration %s: %s", test.MsgInterval, err.Error()) return } } if test.PprofFile != "" { profFile, err := os.Create(test.PprofFile) if err != nil { client.LogError.Fatalln(err) } pprof.StartCPUProfile(profFile) defer pprof.StopCPUProfile() } if test.MaxMessageSize > 0 { message.SetMaxMessageSize(test.MaxMessageSize) } var sender *client.NetworkSender var err error if test.UseTls { var goTlsConfig *tls.Config goTlsConfig, err = tcp.CreateGoTlsConfig(&test.Tls) if err != nil { client.LogError.Fatalf("Error creating TLS config: %s\n", err) } sender, err = client.NewTlsSender(test.Sender, test.IpAddress, goTlsConfig) } else { sender, err = client.NewNetworkSender(test.Sender, test.IpAddress) } if err != nil { client.LogError.Fatalf("Error creating sender: %s\n", err) } unsignedEncoder := client.NewProtobufEncoder(nil) signedEncoder := client.NewProtobufEncoder(&test.Signer) oversizedEncoder := &OversizedEncoder{} var numTestMessages = 1 var unsignedMessages [][]byte var signedMessages [][]byte var oversizedMessages [][]byte rdm := &randomDataMaker{ src: rand.NewSource(time.Now().UnixNano()), asciiOnly: test.AsciiOnly, } if test.VariableSizeMessages { numTestMessages = 64 unsignedMessages = makeVariableMessage(unsignedEncoder, numTestMessages, rdm, false) signedMessages = makeVariableMessage(signedEncoder, numTestMessages, rdm, false) oversizedMessages = makeVariableMessage(oversizedEncoder, 1, rdm, true) } else { if test.StaticMessageSize == 0 { test.StaticMessageSize = 1000 } unsignedMessages = makeFixedMessage(unsignedEncoder, test.StaticMessageSize, rdm) signedMessages = makeFixedMessage(signedEncoder, test.StaticMessageSize, rdm) } // wait for sigint sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT) var msgsSent, bytesSent, msgsDelivered uint64 var corruptPercentage, lastCorruptPercentage, signedPercentage, lastSignedPercentage, oversizedPercentage, lastOversizedPercentage float64 var corrupt bool // set up counter loop ticker := time.NewTicker(time.Duration(time.Second)) go timerLoop(&msgsSent, &bytesSent, ticker) test.CorruptPercentage /= 100.0 test.SignedPercentage /= 100.0 test.OversizedPercentage /= 100.0 var buf []byte for gotsigint := false; !gotsigint; { runtime.Gosched() select { case <-sigChan: gotsigint = true continue default: } msgId := rand.Int() % numTestMessages corruptPercentage = math.Floor(float64(msgsSent) * test.CorruptPercentage) if corruptPercentage != lastCorruptPercentage { lastCorruptPercentage = corruptPercentage corrupt = true } else { corrupt = false } signedPercentage = math.Floor(float64(msgsSent) * test.SignedPercentage) if signedPercentage != lastSignedPercentage { lastSignedPercentage = signedPercentage buf = signedMessages[msgId] } else { oversizedPercentage = math.Floor(float64(msgsSent) * test.OversizedPercentage) if oversizedPercentage != lastOversizedPercentage { lastOversizedPercentage = oversizedPercentage buf = oversizedMessages[0] } else { buf = unsignedMessages[msgId] } } bytesSent += uint64(len(buf)) if err = sendMessage(sender, buf, corrupt); err != nil { client.LogError.Printf("Error sending message: %s\n", err.Error()) } else { msgsDelivered++ } msgsSent++ if test.NumMessages != 0 && msgsSent >= test.NumMessages { break } if test.msgInterval != 0 { time.Sleep(test.msgInterval) } } sender.Close() client.LogInfo.Println("Clean shutdown: ", msgsSent, " messages sent; ", msgsDelivered, " messages delivered.") }
func main() { configFile := flag.String("config", "flood.toml", "Flood configuration file") configTest := flag.String("test", "default", "Test section to load") flag.Parse() if flag.NFlag() == 0 { flag.PrintDefaults() os.Exit(0) } var config FloodConfig if _, err := toml.DecodeFile(*configFile, &config); err != nil { log.Printf("Error decoding config file: %s", err) return } var test FloodTest var ok bool if test, ok = config[*configTest]; !ok { log.Printf("Configuration test: '%s' was not found", *configTest) return } if test.PprofFile != "" { profFile, err := os.Create(test.PprofFile) if err != nil { log.Fatalln(err) } pprof.StartCPUProfile(profFile) defer pprof.StopCPUProfile() } sender, err := client.NewNetworkSender(test.Sender, test.IpAddress) if err != nil { log.Fatalf("Error creating sender: %s\n", err.Error()) } var unsignedEncoder client.Encoder var signedEncoder client.Encoder switch test.Encoder { case "json": unsignedEncoder = client.NewJsonEncoder(nil) signedEncoder = client.NewJsonEncoder(&test.Signer) case "protobuf": unsignedEncoder = client.NewProtobufEncoder(nil) signedEncoder = client.NewProtobufEncoder(&test.Signer) } var numTestMessages = 1 var unsignedMessages [][]byte var signedMessages [][]byte rdm := &randomDataMaker{ src: rand.NewSource(time.Now().UnixNano()), asciiOnly: test.AsciiOnly, } if test.VariableSizeMessages { numTestMessages = 64 unsignedMessages = makeVariableMessage(unsignedEncoder, numTestMessages, rdm) signedMessages = makeVariableMessage(signedEncoder, numTestMessages, rdm) } else { if test.StaticMessageSize == 0 { test.StaticMessageSize = 1000 } unsignedMessages = makeFixedMessage(unsignedEncoder, test.StaticMessageSize, rdm) signedMessages = makeFixedMessage(signedEncoder, test.StaticMessageSize, rdm) } // wait for sigint sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT) var msgsSent, bytesSent uint64 var corruptPercentage, lastCorruptPercentage, signedPercentage, lastSignedPercentage float64 var corrupt bool // set up counter loop ticker := time.NewTicker(time.Duration(time.Second)) go timerLoop(&msgsSent, &bytesSent, ticker) test.CorruptPercentage /= 100.0 test.SignedPercentage /= 100.0 var buf []byte for gotsigint := false; !gotsigint; { runtime.Gosched() select { case <-sigChan: gotsigint = true continue default: } msgId := rand.Int() % numTestMessages corruptPercentage = math.Floor(float64(msgsSent) * test.CorruptPercentage) if corruptPercentage != lastCorruptPercentage { lastCorruptPercentage = corruptPercentage corrupt = true } else { corrupt = false } signedPercentage = math.Floor(float64(msgsSent) * test.SignedPercentage) if signedPercentage != lastSignedPercentage { lastSignedPercentage = signedPercentage buf = signedMessages[msgId] } else { buf = unsignedMessages[msgId] } bytesSent += uint64(len(buf)) err = sendMessage(sender, buf, corrupt) if err != nil { if !strings.Contains(err.Error(), "connection refused") { log.Printf("Error sending message: %s\n", err.Error()) } } else { msgsSent++ if test.NumMessages != 0 && msgsSent >= test.NumMessages { break } } } sender.Close() log.Println("Clean shutdown: ", msgsSent, " messages sent") }