func LoadHekadConfig(configPath string) (config *HekadConfig, err error) {
idle, _ := time.ParseDuration("2m")
config = &HekadConfig{Maxprocs: 1,
PoolSize: 100,
DecoderPoolSize: 4,
ChanSize: 50,
CpuProfName: "",
MemProfName: "",
MaxMsgLoops: 4,
MaxMsgProcessInject: 1,
MaxMsgProcessDuration: 100000,
MaxMsgTimerInject: 10,
MaxPackIdle: idle,
BaseDir: filepath.FromSlash("/var/cache/hekad"),
ShareDir: filepath.FromSlash("/usr/share/heka"),
}
var configFile map[string]toml.Primitive
p, err := os.Open(configPath)
if err != nil {
return nil, fmt.Errorf("Error opening config file: %s", err)
}
fi, err := p.Stat()
if err != nil {
return nil, fmt.Errorf("Error fetching config file info: %s", err)
}
if fi.IsDir() {
files, _ := ioutil.ReadDir(configPath)
for _, f := range files {
fName := f.Name()
if strings.HasPrefix(fName, ".") || strings.HasSuffix(fName, ".bak") ||
strings.HasSuffix(fName, ".tmp") || strings.HasSuffix(fName, "~") {
// Skip obviously non-relevant files.
continue
}
fPath := filepath.Join(configPath, fName)
if _, err = toml.DecodeFile(fPath, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
}
} else {
if _, err = toml.DecodeFile(configPath, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
}
empty_ignore := map[string]interface{}{}
parsed_config, ok := configFile["hekad"]
if ok {
if err = toml.PrimitiveDecodeStrict(parsed_config, config, empty_ignore); err != nil {
err = fmt.Errorf("Can't unmarshal config: %s", err)
}
}
return
}
func main() {
configFile := flag.String("config", "logstreamer.toml", "Heka Logstreamer configuration file")
flag.Parse()
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
p, err := os.Open(*configFile)
if err != nil {
client.LogError.Fatalf("Error opening config file: %s", err)
}
fi, err := p.Stat()
if err != nil {
client.LogError.Fatalf("Error fetching config file info: %s", err)
}
fconfig := make(FileConfig)
if fi.IsDir() {
files, _ := ioutil.ReadDir(*configFile)
for _, f := range files {
fName := f.Name()
if strings.HasPrefix(fName, ".") || strings.HasSuffix(fName, ".bak") ||
strings.HasSuffix(fName, ".tmp") || strings.HasSuffix(fName, "~") {
// Skip obviously non-relevant files.
continue
}
fPath := filepath.Join(*configFile, fName)
if _, err = toml.DecodeFile(fPath, &fconfig); err != nil {
client.LogError.Fatalf("Error decoding config file: %s", err)
}
}
} else {
if _, err := toml.DecodeFile(*configFile, &fconfig); err != nil {
client.LogError.Fatalf("Error decoding config file: %s", err)
}
}
// Filter out logstream inputs
inputs := make(map[string]toml.Primitive)
for name, prim := range fconfig {
basic := new(Basic)
if name == "LogstreamerInput" {
inputs[name] = prim
} else if err := toml.PrimitiveDecode(prim, &basic); err == nil {
if basic.PluginType == "LogstreamerInput" {
inputs[name] = prim
}
}
}
// Go through the logstreams and parse their configs
for name, prim := range inputs {
parseConfig(name, prim)
}
}
func main() {
configFile := flag.String("config", "logstreamer.toml", "Heka Logstreamer configuration file")
flag.Parse()
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
fconfig := make(FileConfig)
if _, err := toml.DecodeFile(*configFile, &fconfig); err != nil {
log.Printf("Error decoding config file: %s", err)
return
}
// Filter out logstream inputs
inputs := make(map[string]toml.Primitive)
for name, prim := range fconfig {
basic := new(Basic)
if name == "LogstreamerInput" {
inputs[name] = prim
} else if err := toml.PrimitiveDecode(prim, &basic); err == nil {
if basic.PluginType == "LogstreamerInput" {
inputs[name] = prim
}
}
}
// Go through the logstreams and parse their configs
for name, prim := range inputs {
parseConfig(name, prim)
}
}
func (pdi *ProcessDirectoryInput) loadProcessFile(path string) (*ProcessEntry, error) {
var err error
unparsedConfig := make(map[string]toml.Primitive)
if _, err = toml.DecodeFile(path, &unparsedConfig); err != nil {
return nil, err
}
section, ok := unparsedConfig["ProcessInput"]
if !ok {
err = errors.New("No `ProcessInput` section.")
return nil, err
}
maker, err := NewPluginMaker("ProcessInput", pdi.pConfig, section)
if err != nil {
return nil, fmt.Errorf("can't create plugin maker: %s", err)
}
mutMaker := maker.(MutableMaker)
mutMaker.SetName(path)
if err = mutMaker.PrepConfig(); err != nil {
return nil, fmt.Errorf("can't prep config: %s", err)
}
entry := &ProcessEntry{
maker: mutMaker,
config: mutMaker.Config().(*ProcessInputConfig),
}
return entry, nil
}
// On Heka restarts this function reloads all previously running SandboxFilters
// using the script, configuration, and preservation files in the working
// directory.
func (this *SandboxManagerFilter) restoreSandboxes(fr pipeline.FilterRunner, h pipeline.PluginHelper, dir string) {
glob := fmt.Sprintf("%s-*.toml", getNormalizedName(fr.Name()))
if matches, err := filepath.Glob(filepath.Join(dir, glob)); err == nil {
for _, fn := range matches {
var configFile pipeline.ConfigFile
if _, err = toml.DecodeFile(fn, &configFile); err != nil {
fr.LogError(fmt.Errorf("restoreSandboxes failed: %s\n", err))
continue
} else {
for _, conf := range configFile {
var runner pipeline.FilterRunner
name := path.Base(fn[:len(fn)-5])
fr.LogMessage(fmt.Sprintf("Loading: %s", name))
runner, err = this.createRunner(dir, name, conf)
if err != nil {
fr.LogError(fmt.Errorf("createRunner failed: %s\n", err.Error()))
removeAll(dir, fmt.Sprintf("%s.*", name))
break
}
err = h.PipelineConfig().AddFilterRunner(runner)
if err != nil {
fr.LogError(err)
} else {
atomic.AddInt32(&this.currentFilters, 1)
}
break // only interested in the first item
}
}
}
}
}
// LoadFromConfigFile loads a TOML configuration file and stores the
// result in the value pointed to by config. The maps in the config
// will be initialized as needed.
//
// The PipelineConfig should be already initialized before passed in via
// its Init function.
func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) {
var configFile ConfigFile
if _, err = toml.DecodeFile(filename, &configFile); err != nil {
return fmt.Errorf("Error decoding config file: %s", err)
}
// Load all the plugins
var errcnt uint
for name, conf := range configFile {
if name == "hekad" {
continue
}
log.Printf("Loading: [%s]\n", name)
errcnt += self.loadSection(name, conf)
}
// Add JSON/PROTOCOL_BUFFER decoders if none were configured
var configDefault ConfigFile
toml.Decode(defaultDecoderTOML, &configDefault)
dWrappers := self.DecoderWrappers
if _, ok := dWrappers["ProtobufDecoder"]; !ok {
log.Println("Loading: [ProtobufDecoder]")
errcnt += self.loadSection("ProtobufDecoder", configDefault["ProtobufDecoder"])
}
if errcnt != 0 {
return fmt.Errorf("%d errors loading plugins", errcnt)
}
return
}
func LoadHekadConfig(filename string) (config *HekadConfig, err error) {
config = &HekadConfig{Maxprocs: 1,
PoolSize: 100,
DecoderPoolSize: 4,
ChanSize: 50,
CpuProfName: "",
MemProfName: "",
MaxMsgLoops: 4,
MaxMsgProcessInject: 1,
MaxMsgTimerInject: 10,
}
var configFile map[string]toml.Primitive
if _, err = toml.DecodeFile(filename, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
empty_ignore := map[string]interface{}{}
parsed_config, ok := configFile["hekad"]
if ok {
if err = toml.PrimitiveDecodeStrict(parsed_config, &config, empty_ignore); err != nil {
err = fmt.Errorf("Can't unmarshal config: %s", err)
}
}
return
}
func main() {
configFile := flag.String("config", "sbmgr.toml", "Sandbox manager configuration file")
scriptFile := flag.String("script", "xyz.lua", "Sandbox script file")
scriptConfig := flag.String("scriptconfig", "xyz.toml", "Sandbox script configuration file")
filterName := flag.String("filtername", "filter", "Sandbox filter name (used on unload)")
action := flag.String("action", "load", "Sandbox manager action")
flag.Parse()
var config SbmgrConfig
if _, err := toml.DecodeFile(*configFile, &config); err != nil {
log.Printf("Error decoding config file: %s", err)
return
}
sender, err := client.NewNetworkSender("tcp", config.IpAddress)
if err != nil {
log.Fatalf("Error creating sender: %s\n", err.Error())
}
encoder := client.NewProtobufEncoder(&config.Signer)
manager := client.NewClient(sender, encoder)
hostname, _ := os.Hostname()
msg := &message.Message{}
msg.SetType("heka.control.sandbox")
msg.SetTimestamp(time.Now().UnixNano())
msg.SetUuid(uuid.NewRandom())
msg.SetHostname(hostname)
switch *action {
case "load":
code, err := ioutil.ReadFile(*scriptFile)
if err != nil {
log.Printf("Error reading scriptFile: %s\n", err.Error())
return
}
msg.SetPayload(string(code))
conf, err := ioutil.ReadFile(*scriptConfig)
if err != nil {
log.Printf("Error reading scriptConfig: %s\n", err.Error())
return
}
f, _ := message.NewField("config", string(conf), "toml")
msg.AddField(f)
case "unload":
f, _ := message.NewField("name", *filterName, "")
msg.AddField(f)
default:
log.Printf("Invalid action: %s", *action)
}
f1, _ := message.NewField("action", *action, "")
msg.AddField(f1)
err = manager.SendMessage(msg)
if err != nil {
log.Printf("Error sending message: %s\n", err.Error())
}
}
// Handles reading a TOML based configuration file, and loading an
// initialized Application, ready to Run
func LoadApplicationFromFileName(filename string, logging int) (
app *Application, err error) {
var configFile ConfigFile
if _, err = toml.DecodeFile(filename, &configFile); err != nil {
return nil, fmt.Errorf("Error decoding config file: %s", err)
}
env := envconf.Load()
return LoadApplication(configFile, env, logging)
}
func main() {
var config Config
config_file := flag.String("config", filepath.FromSlash("/etc/heka-server.toml"), "Config file for log server")
if _, err := toml.DecodeFile(*config_file, &config); err != nil {
fmt.Println(err)
return
}
fmt.Println(config)
}
// LoadHeimdallConfig accepts the path of a configuration file
// to load. Additionally, it configures defaults for values that
// are not specified within the configuration passed.
func LoadHeimdallConfig(path string) (config *Config, err error) {
config = &Config{CleanupInterval: "3m",
Port: ":9001",
Cert: "cert.pem",
Key: "key.pem",
Auth: AuthConfig{DBPath: "/var/cache/heimdall/auth"},
Blacklist: BlacklistConfig{DBPath: "/var/cache/heimdall/blacklist", Path: "/blacklist/", DefaultTTL: "1h"},
Whitelist: WhitelistConfig{DBPath: "/var/cache/heimdall/whitelist", Path: "/whitelist/", DefaultTTL: "72h"},
PanOS: PanOSConfig{Enabled: false, Path: "/panos/"}}
if _, err := toml.DecodeFile(path, config); err != nil {
return nil, fmt.Errorf("Problem decoding configuration file: %s", err)
}
return config, nil
}
// LoadFromConfigFile loads a TOML configuration file and stores the
// result in the value pointed to by config. The maps in the config
// will be initialized as needed.
//
// The PipelineConfig should be already initialized before passed in via
// its Init function.
func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) {
var configFile ConfigFile
if _, err = toml.DecodeFile(filename, &configFile); err != nil {
return fmt.Errorf("Error decoding config file: %s", err)
}
// Load all the plugins
var errcnt uint
for name, conf := range configFile {
log.Println("Loading: ", name)
errcnt += self.loadSection(name, conf)
}
// Add JSON/PROTOCOL_BUFFER decoders if none were configured
var configDefault ConfigFile
toml.Decode(defaultDecoderTOML, &configDefault)
dWrappers := self.DecoderWrappers
if _, ok := dWrappers["JsonDecoder"]; !ok {
log.Println("Loading: JsonDecoder")
errcnt += self.loadSection("JsonDecoder", configDefault["JsonDecoder"])
}
if _, ok := dWrappers["ProtobufDecoder"]; !ok {
log.Println("Loading: ProtobufDecoder")
errcnt += self.loadSection("ProtobufDecoder", configDefault["ProtobufDecoder"])
}
// Create / prep the DecoderSet pool
var dRunner DecoderRunner
for i := 0; i < Globals().DecoderPoolSize; i++ {
if self.DecoderSets[i], err = newDecoderSet(dWrappers); err != nil {
log.Println(err)
errcnt += 1
}
for _, dRunner = range self.DecoderSets[i].AllByName() {
dRunner.Start(self, &self.decodersWg)
}
self.decodersChan <- self.DecoderSets[i]
}
if errcnt != 0 {
return fmt.Errorf("%d errors loading plugins", errcnt)
}
return
}
func (this *SandboxManagerFilter) Init(config interface{}) (err error) {
conf := config.(*SandboxManagerFilterConfig)
this.maxFilters = conf.MaxFilters
this.workingDirectory, _ = filepath.Abs(conf.WorkingDirectory)
if err = os.MkdirAll(this.workingDirectory, 0700); err != nil {
return err
}
filename := path.Join(this.workingDirectory, "config.toml")
_, err = os.Stat(filename)
if err == nil {
var configFile ConfigFile
if _, err := toml.DecodeFile(filename, &configFile); err != nil {
return fmt.Errorf("Error decoding config file: %s", err)
}
}
return nil
}
func (pdi *ProcessDirectoryInput) loadProcessFile(path string) (*ProcessEntry, error) {
var err error
unparsedConfig := make(map[string]toml.Primitive)
if _, err = toml.DecodeFile(path, &unparsedConfig); err != nil {
return nil, err
}
section, ok := unparsedConfig["ProcessInput"]
if !ok {
err = errors.New("No `ProcessInput` section.")
return nil, err
}
maker, err := NewPluginMaker("ProcessInput", pdi.pConfig, section)
if err != nil {
return nil, fmt.Errorf("can't create plugin maker: %s", err)
}
mutMaker := maker.(MutableMaker)
mutMaker.SetName(path)
prepCommonTypedConfig := func() (interface{}, error) {
commonTypedConfig, err := mutMaker.OrigPrepCommonTypedConfig()
if err != nil {
return nil, err
}
commonInput := commonTypedConfig.(CommonInputConfig)
commonInput.Retries = RetryOptions{
MaxDelay: "30s",
Delay: "250ms",
MaxRetries: -1,
}
if commonInput.CanExit == nil {
b := true
commonInput.CanExit = &b
}
return commonInput, nil
}
mutMaker.SetPrepCommonTypedConfig(prepCommonTypedConfig)
entry := &ProcessEntry{
maker: mutMaker,
}
return entry, nil
}
func (pdi *ProcessDirectoryInput) loadProcessFile(path string) (config *ProcessInputConfig,
err error) {
unparsedConfig := make(map[string]toml.Primitive)
if _, err = toml.DecodeFile(path, &unparsedConfig); err != nil {
return
}
section, ok := unparsedConfig["ProcessInput"]
if !ok {
err = errors.New("No `ProcessInput` section.")
return
}
config = &ProcessInputConfig{
ParserType: "token",
ParseStdout: true,
Trim: true,
}
if err = toml.PrimitiveDecodeStrict(section, config, nil); err != nil {
return nil, err
}
return
}
func main() {
configFile := flag.String("config", "sbmgr.toml", "Sandbox manager configuration file")
scriptFile := flag.String("script", "xyz.lua", "Sandbox script file")
scriptConfig := flag.String("scriptconfig", "xyz.toml", "Sandbox script configuration file")
filterName := flag.String("filtername", "filter", "Sandbox filter name (used on unload)")
action := flag.String("action", "load", "Sandbox manager action")
flag.Parse()
var config SbmgrConfig
if _, err := toml.DecodeFile(*configFile, &config); err != nil {
client.LogError.Printf("Error decoding config file: %s", err)
return
}
var sender *client.NetworkSender
var err error
if config.UseTls {
var goTlsConfig *tls.Config
goTlsConfig, err = tcp.CreateGoTlsConfig(&config.Tls)
if err != nil {
client.LogError.Fatalf("Error creating TLS config: %s\n", err)
}
sender, err = client.NewTlsSender("tcp", config.IpAddress, goTlsConfig)
} else {
sender, err = client.NewNetworkSender("tcp", config.IpAddress)
}
if err != nil {
client.LogError.Fatalf("Error creating sender: %s\n", err.Error())
}
encoder := client.NewProtobufEncoder(&config.Signer)
manager := client.NewClient(sender, encoder)
hostname, _ := os.Hostname()
msg := &message.Message{}
msg.SetLogger(pipeline.HEKA_DAEMON) // identify the message as 'internal' for filtering purposes
msg.SetType("heka.control.sandbox")
msg.SetTimestamp(time.Now().UnixNano())
msg.SetUuid(uuid.NewRandom())
msg.SetHostname(hostname)
switch *action {
case "load":
code, err := ioutil.ReadFile(*scriptFile)
if err != nil {
client.LogError.Printf("Error reading scriptFile: %s\n", err.Error())
return
}
msg.SetPayload(string(code))
conf, err := ioutil.ReadFile(*scriptConfig)
if err != nil {
client.LogError.Printf("Error reading scriptConfig: %s\n", err.Error())
return
}
f, _ := message.NewField("config", string(conf), "toml")
msg.AddField(f)
case "unload":
f, _ := message.NewField("name", *filterName, "")
msg.AddField(f)
default:
client.LogError.Printf("Invalid action: %s", *action)
}
f1, _ := message.NewField("action", *action, "")
msg.AddField(f1)
err = manager.SendMessage(msg)
if err != nil {
client.LogError.Printf("Error sending message: %s\n", err.Error())
}
}
// Loads all plugin configuration from a TOML configuration file. The
// PipelineConfig should be already initialized via the Init function before
// this method is called.
func (self *PipelineConfig) LoadFromConfigFile(filename string) (err error) {
var configFile ConfigFile
if _, err = toml.DecodeFile(filename, &configFile); err != nil {
return fmt.Errorf("Error decoding config file: %s", err)
}
var (
errcnt uint
protobufDRegistered bool
protobufERegistered bool
)
sectionsByCategory := make(map[string][]*ConfigSection)
// Load all the plugin globals and file them by category.
for name, conf := range configFile {
if name == HEKA_DAEMON {
continue
}
log.Printf("Pre-loading: [%s]\n", name)
section := &ConfigSection{
name: name,
tomlSection: conf,
}
if err = self.loadPluginGlobals(section); err != nil {
self.log(err.Error())
errcnt++
continue
}
category := getPluginCategory(section.globals.Typ)
if category == "" {
self.log(fmt.Sprintf("Type doesn't contain valid plugin name: %s\n",
section.globals.Typ))
errcnt++
continue
}
section.category = category
if section.globals.Typ == "MultiDecoder" {
// Special case MultiDecoders so we can make sure they get
// registered *after* all possible subdecoders.
sectionsByCategory["MultiDecoder"] = append(sectionsByCategory["MultiDecoder"],
section)
} else {
sectionsByCategory[category] = append(sectionsByCategory[category], section)
}
if name == "ProtobufDecoder" {
protobufDRegistered = true
}
if name == "ProtobufEncoder" {
protobufERegistered = true
}
}
// Make sure ProtobufDecoder is registered.
if !protobufDRegistered {
var configDefault ConfigFile
toml.Decode(protobufDecoderToml, &configDefault)
log.Println("Pre-loading: [ProtobufDecoder]")
section := &ConfigSection{
name: "ProtobufDecoder",
category: "Decoder",
tomlSection: configDefault["ProtobufDecoder"],
}
if err = self.loadPluginGlobals(section); err != nil {
// This really shouldn't happen.
self.log(err.Error())
errcnt++
} else {
sectionsByCategory["Decoder"] = append(sectionsByCategory["Decoder"],
section)
}
}
// Make sure ProtobufEncoder is registered.
if !protobufERegistered {
var configDefault ConfigFile
toml.Decode(protobufEncoderToml, &configDefault)
log.Println("Pre-loading: [ProtobufEncoder]")
section := &ConfigSection{
name: "ProtobufEncoder",
category: "Encoder",
tomlSection: configDefault["ProtobufEncoder"],
}
if err = self.loadPluginGlobals(section); err != nil {
// This really shouldn't happen.
self.log(err.Error())
errcnt++
} else {
sectionsByCategory["Encoder"] = append(sectionsByCategory["Encoder"],
section)
}
}
multiDecoders := make([]multiDecoderNode, len(sectionsByCategory["MultiDecoder"]))
multiConfigs := make(map[string]*ConfigSection)
for i, section := range sectionsByCategory["MultiDecoder"] {
multiConfigs[section.name] = section
multiDecoders[i] = newMultiDecoderNode(section.name, subsFromSection(section.tomlSection))
}
multiDecoders, err = orderDependencies(multiDecoders)
if err != nil {
return err
}
for i, d := range multiDecoders {
sectionsByCategory["MultiDecoder"][i] = multiConfigs[d.name]
}
// Append MultiDecoders to the end of the Decoders list.
sectionsByCategory["Decoder"] = append(sectionsByCategory["Decoder"],
sectionsByCategory["MultiDecoder"]...)
// Force decoders and encoders to be registered before the other plugin
// types are initialized so we know they'll be there for inputs and
// outputs to use during initialization.
order := []string{"Decoder", "Encoder", "Input", "Filter", "Output"}
for _, category := range order {
for _, section := range sectionsByCategory[category] {
log.Printf("Loading: [%s]\n", section.name)
if err = self.loadSection(section); err != nil {
self.log(err.Error())
errcnt++
}
}
}
if errcnt != 0 {
return fmt.Errorf("%d errors loading plugins", errcnt)
}
return
}
func main() {
configFile := flag.String("config", "sbmgrload.toml", "Sandbox manager load configuration file")
action := flag.String("action", "load", "load/unload")
numItems := flag.Int("num", 1, "Number of sandboxes to load/unload")
flag.Parse()
code := `
lastTime = os.time() * 1e9
lastCount = 0
count = 0
rate = 0.0
rates = {}
function process_message ()
count = count + 1
return 0
end
function timer_event(ns)
local msgsSent = count - lastCount
if msgsSent == 0 then return end
local elapsedTime = ns - lastTime
if elapsedTime == 0 then return end
lastCount = count
lastTime = ns
rate = msgsSent / (elapsedTime / 1e9)
rates[#rates+1] = rate
output(string.format("Got %d messages. %0.2f msg/sec", count, rate))
inject_message()
local samples = #rates
if samples == 10 then -- generate a summary every 10 samples
table.sort(rates)
local min = rates[1]
local max = rates[samples]
local sum = 0
for i, val in ipairs(rates) do
sum = sum + val
end
output(string.format("AGG Sum. Min: %0.2f Max: %0.2f Mean: %0.2f", min, max, sum/samples))
inject_message()
rates = {}
end
end
`
confFmt := `
[CounterSandbox%d]
type = "SandboxFilter"
message_matcher = "Type == 'hekabench'"
ticker_interval = 1.0
[CounterSandbox%d.settings]
type = "lua"
filename = ""
preserve_data = true
memory_limit = 32767
instruction_limit = 1000
output_limit = 1024
`
var config SbmgrConfig
if _, err := toml.DecodeFile(*configFile, &config); err != nil {
log.Printf("Error decoding config file: %s", err)
return
}
sender, err := client.NewNetworkSender("tcp", config.IpAddress)
if err != nil {
log.Fatalf("Error creating sender: %s\n", err.Error())
}
encoder := client.NewProtobufEncoder(&config.Signer)
manager := client.NewClient(sender, encoder)
hostname, _ := os.Hostname()
switch *action {
case "load":
for i := 0; i < *numItems; i++ {
conf := fmt.Sprintf(confFmt, i, i)
msg := &message.Message{}
msg.SetType("heka.control.sandbox")
msg.SetTimestamp(time.Now().UnixNano())
msg.SetUuid(uuid.NewRandom())
msg.SetHostname(hostname)
msg.SetPayload(code)
f, _ := message.NewField("config", conf, message.Field_RAW)
msg.AddField(f)
f1, _ := message.NewField("action", *action, message.Field_RAW)
msg.AddField(f1)
err = manager.SendMessage(msg)
if err != nil {
log.Printf("Error sending message: %s\n", err.Error())
}
}
case "unload":
for i := 0; i < *numItems; i++ {
msg := &message.Message{}
msg.SetType("heka.control.sandbox")
msg.SetTimestamp(time.Now().UnixNano())
msg.SetUuid(uuid.NewRandom())
msg.SetHostname(hostname)
msg.SetPayload(string(code))
f, _ := message.NewField("name", fmt.Sprintf("CounterSandbox%d", i), message.Field_RAW)
msg.AddField(f)
f1, _ := message.NewField("action", *action, message.Field_RAW)
msg.AddField(f1)
err = manager.SendMessage(msg)
if err != nil {
log.Printf("Error sending message: %s\n", err.Error())
}
}
default:
log.Printf("Invalid action: %s\n", *action)
}
}
func main() {
configFile := flag.String("config", "sbmgrload.toml", "Sandbox manager load configuration file")
action := flag.String("action", "load", "load/unload")
numItems := flag.Int("num", 1, "Number of sandboxes to load/unload")
flag.Parse()
code := `
require "string"
require "table"
require "os"
lastTime = os.time() * 1e9
lastCount = 0
count = 0
rate = 0.0
rates = {}
function process_message ()
count = count + 1
return 0
end
function timer_event(ns)
local msgsSent = count - lastCount
if msgsSent == 0 then return end
local elapsedTime = ns - lastTime
if elapsedTime == 0 then return end
lastCount = count
lastTime = ns
rate = msgsSent / (elapsedTime / 1e9)
rates[#rates+1] = rate
inject_payload("txt", "rate", string.format("Got %d messages. %0.2f msg/sec", count, rate))
local samples = #rates
if samples == 10 then -- generate a summary every 10 samples
table.sort(rates)
local min = rates[1]
local max = rates[samples]
local sum = 0
for i, val in ipairs(rates) do
sum = sum + val
end
inject_payload("txt", "summary", string.format("AGG Sum. Min: %0.2f Max: %0.2f Mean: %0.2f", min, max, sum/samples))
rates = {}
end
end
`
confFmt := `
[CounterSandbox%d]
type = "SandboxFilter"
message_matcher = "Type == 'hekabench'"
ticker_interval = 1
filename = ""
preserve_data = true
`
var config SbmgrConfig
if _, err := toml.DecodeFile(*configFile, &config); err != nil {
client.LogError.Printf("Error decoding config file: %s", err)
return
}
var sender *client.NetworkSender
var err error
if config.UseTls {
var goTlsConfig *tls.Config
goTlsConfig, err = tcp.CreateGoTlsConfig(&config.Tls)
if err != nil {
client.LogError.Fatalf("Error creating TLS config: %s\n", err)
}
sender, err = client.NewTlsSender("tcp", config.IpAddress, goTlsConfig)
} else {
sender, err = client.NewNetworkSender("tcp", config.IpAddress)
}
if err != nil {
client.LogError.Fatalf("Error creating sender: %s\n", err.Error())
}
encoder := client.NewProtobufEncoder(&config.Signer)
manager := client.NewClient(sender, encoder)
hostname, _ := os.Hostname()
switch *action {
case "load":
for i := 0; i < *numItems; i++ {
conf := fmt.Sprintf(confFmt, i)
msg := &message.Message{}
msg.SetLogger(pipeline.HEKA_DAEMON)
msg.SetType("heka.control.sandbox")
msg.SetTimestamp(time.Now().UnixNano())
msg.SetUuid(uuid.NewRandom())
msg.SetHostname(hostname)
msg.SetPayload(code)
f, _ := message.NewField("config", conf, "toml")
msg.AddField(f)
f1, _ := message.NewField("action", *action, "")
msg.AddField(f1)
err = manager.SendMessage(msg)
if err != nil {
client.LogError.Printf("Error sending message: %s\n", err.Error())
}
}
case "unload":
for i := 0; i < *numItems; i++ {
msg := &message.Message{}
msg.SetLogger(pipeline.HEKA_DAEMON)
msg.SetType("heka.control.sandbox")
msg.SetTimestamp(time.Now().UnixNano())
msg.SetUuid(uuid.NewRandom())
msg.SetHostname(hostname)
f, _ := message.NewField("name", fmt.Sprintf("CounterSandbox%d", i), "")
msg.AddField(f)
f1, _ := message.NewField("action", *action, "")
msg.AddField(f1)
err = manager.SendMessage(msg)
if err != nil {
client.LogError.Printf("Error sending message: %s\n", err.Error())
}
}
default:
client.LogError.Printf("Invalid action: %s\n", *action)
}
}
func main() {
configFile := flag.String("config", "flood.toml", "Heka Flood configuration file")
configTest := flag.String("test", "default", "Test section to load")
flag.Parse()
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
var config FloodConfig
if _, err := toml.DecodeFile(*configFile, &config); err != nil {
client.LogError.Printf("Error decoding config file: %s", err)
return
}
var test FloodTest
var ok bool
if test, ok = config[*configTest]; !ok {
client.LogError.Printf("Configuration test: '%s' was not found", *configTest)
return
}
if test.MsgInterval != "" {
var err error
if test.msgInterval, err = time.ParseDuration(test.MsgInterval); err != nil {
client.LogError.Printf("Invalid message_interval duration %s: %s", test.MsgInterval,
err.Error())
return
}
}
if test.PprofFile != "" {
profFile, err := os.Create(test.PprofFile)
if err != nil {
client.LogError.Fatalln(err)
}
pprof.StartCPUProfile(profFile)
defer pprof.StopCPUProfile()
}
if test.MaxMessageSize > 0 {
message.SetMaxMessageSize(test.MaxMessageSize)
}
var sender *client.NetworkSender
var err error
if test.UseTls {
var goTlsConfig *tls.Config
goTlsConfig, err = tcp.CreateGoTlsConfig(&test.Tls)
if err != nil {
client.LogError.Fatalf("Error creating TLS config: %s\n", err)
}
sender, err = client.NewTlsSender(test.Sender, test.IpAddress, goTlsConfig)
} else {
sender, err = client.NewNetworkSender(test.Sender, test.IpAddress)
}
if err != nil {
client.LogError.Fatalf("Error creating sender: %s\n", err)
}
unsignedEncoder := client.NewProtobufEncoder(nil)
signedEncoder := client.NewProtobufEncoder(&test.Signer)
oversizedEncoder := &OversizedEncoder{}
var numTestMessages = 1
var unsignedMessages [][]byte
var signedMessages [][]byte
var oversizedMessages [][]byte
rdm := &randomDataMaker{
src: rand.NewSource(time.Now().UnixNano()),
asciiOnly: test.AsciiOnly,
}
if test.VariableSizeMessages {
numTestMessages = 64
unsignedMessages = makeVariableMessage(unsignedEncoder, numTestMessages, rdm, false)
signedMessages = makeVariableMessage(signedEncoder, numTestMessages, rdm, false)
oversizedMessages = makeVariableMessage(oversizedEncoder, 1, rdm, true)
} else {
if test.StaticMessageSize == 0 {
test.StaticMessageSize = 1000
}
unsignedMessages = makeFixedMessage(unsignedEncoder, test.StaticMessageSize,
rdm)
signedMessages = makeFixedMessage(signedEncoder, test.StaticMessageSize,
rdm)
}
// wait for sigint
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT)
var msgsSent, bytesSent, msgsDelivered uint64
var corruptPercentage, lastCorruptPercentage, signedPercentage, lastSignedPercentage, oversizedPercentage, lastOversizedPercentage float64
var corrupt bool
// set up counter loop
ticker := time.NewTicker(time.Duration(time.Second))
go timerLoop(&msgsSent, &bytesSent, ticker)
test.CorruptPercentage /= 100.0
test.SignedPercentage /= 100.0
test.OversizedPercentage /= 100.0
var buf []byte
for gotsigint := false; !gotsigint; {
runtime.Gosched()
select {
case <-sigChan:
gotsigint = true
continue
default:
}
msgId := rand.Int() % numTestMessages
corruptPercentage = math.Floor(float64(msgsSent) * test.CorruptPercentage)
if corruptPercentage != lastCorruptPercentage {
lastCorruptPercentage = corruptPercentage
corrupt = true
} else {
corrupt = false
}
signedPercentage = math.Floor(float64(msgsSent) * test.SignedPercentage)
if signedPercentage != lastSignedPercentage {
lastSignedPercentage = signedPercentage
buf = signedMessages[msgId]
} else {
oversizedPercentage = math.Floor(float64(msgsSent) * test.OversizedPercentage)
if oversizedPercentage != lastOversizedPercentage {
lastOversizedPercentage = oversizedPercentage
buf = oversizedMessages[0]
} else {
buf = unsignedMessages[msgId]
}
}
bytesSent += uint64(len(buf))
if err = sendMessage(sender, buf, corrupt); err != nil {
client.LogError.Printf("Error sending message: %s\n", err.Error())
} else {
msgsDelivered++
}
msgsSent++
if test.NumMessages != 0 && msgsSent >= test.NumMessages {
break
}
if test.msgInterval != 0 {
time.Sleep(test.msgInterval)
}
}
sender.Close()
client.LogInfo.Println("Clean shutdown: ", msgsSent, " messages sent; ",
msgsDelivered, " messages delivered.")
}
func main() {
configFile := flag.String("config", "flood.toml", "Flood configuration file")
configTest := flag.String("test", "default", "Test section to load")
flag.Parse()
if flag.NFlag() == 0 {
flag.PrintDefaults()
os.Exit(0)
}
var config FloodConfig
if _, err := toml.DecodeFile(*configFile, &config); err != nil {
log.Printf("Error decoding config file: %s", err)
return
}
var test FloodTest
var ok bool
if test, ok = config[*configTest]; !ok {
log.Printf("Configuration test: '%s' was not found", *configTest)
return
}
if test.PprofFile != "" {
profFile, err := os.Create(test.PprofFile)
if err != nil {
log.Fatalln(err)
}
pprof.StartCPUProfile(profFile)
defer pprof.StopCPUProfile()
}
sender, err := client.NewNetworkSender(test.Sender, test.IpAddress)
if err != nil {
log.Fatalf("Error creating sender: %s\n", err.Error())
}
var unsignedEncoder client.Encoder
var signedEncoder client.Encoder
switch test.Encoder {
case "json":
unsignedEncoder = client.NewJsonEncoder(nil)
signedEncoder = client.NewJsonEncoder(&test.Signer)
case "protobuf":
unsignedEncoder = client.NewProtobufEncoder(nil)
signedEncoder = client.NewProtobufEncoder(&test.Signer)
}
var numTestMessages = 1
var unsignedMessages [][]byte
var signedMessages [][]byte
rdm := &randomDataMaker{
src: rand.NewSource(time.Now().UnixNano()),
asciiOnly: test.AsciiOnly,
}
if test.VariableSizeMessages {
numTestMessages = 64
unsignedMessages = makeVariableMessage(unsignedEncoder, numTestMessages, rdm)
signedMessages = makeVariableMessage(signedEncoder, numTestMessages, rdm)
} else {
if test.StaticMessageSize == 0 {
test.StaticMessageSize = 1000
}
unsignedMessages = makeFixedMessage(unsignedEncoder, test.StaticMessageSize,
rdm)
signedMessages = makeFixedMessage(signedEncoder, test.StaticMessageSize,
rdm)
}
// wait for sigint
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT)
var msgsSent, bytesSent uint64
var corruptPercentage, lastCorruptPercentage, signedPercentage, lastSignedPercentage float64
var corrupt bool
// set up counter loop
ticker := time.NewTicker(time.Duration(time.Second))
go timerLoop(&msgsSent, &bytesSent, ticker)
test.CorruptPercentage /= 100.0
test.SignedPercentage /= 100.0
var buf []byte
for gotsigint := false; !gotsigint; {
runtime.Gosched()
select {
case <-sigChan:
gotsigint = true
continue
default:
}
msgId := rand.Int() % numTestMessages
corruptPercentage = math.Floor(float64(msgsSent) * test.CorruptPercentage)
if corruptPercentage != lastCorruptPercentage {
lastCorruptPercentage = corruptPercentage
corrupt = true
} else {
corrupt = false
}
signedPercentage = math.Floor(float64(msgsSent) * test.SignedPercentage)
if signedPercentage != lastSignedPercentage {
lastSignedPercentage = signedPercentage
buf = signedMessages[msgId]
} else {
buf = unsignedMessages[msgId]
}
bytesSent += uint64(len(buf))
err = sendMessage(sender, buf, corrupt)
if err != nil {
if !strings.Contains(err.Error(), "connection refused") {
log.Printf("Error sending message: %s\n",
err.Error())
}
} else {
msgsSent++
if test.NumMessages != 0 && msgsSent >= test.NumMessages {
break
}
}
}
sender.Close()
log.Println("Clean shutdown: ", msgsSent, " messages sent")
}