// Satisfies the `pipeline.ReportingPlugin` interface to provide sandbox state
// information to the Heka report and dashboard.
func (s *SandboxDecoder) ReportMsg(msg *message.Message) error {
if s.sb == nil {
return fmt.Errorf("Decoder is not running")
}
s.reportLock.RLock()
defer s.reportLock.RUnlock()
message.NewIntField(msg, "Memory", int(s.sb.Usage(TYPE_MEMORY,
STAT_CURRENT)), "B")
message.NewIntField(msg, "MaxMemory", int(s.sb.Usage(TYPE_MEMORY,
STAT_MAXIMUM)), "B")
message.NewIntField(msg, "MaxInstructions", int(s.sb.Usage(
TYPE_INSTRUCTIONS, STAT_MAXIMUM)), "count")
message.NewIntField(msg, "MaxOutput", int(s.sb.Usage(TYPE_OUTPUT,
STAT_MAXIMUM)), "B")
message.NewInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&s.processMessageCount), "count")
message.NewInt64Field(msg, "ProcessMessageFailures", atomic.LoadInt64(&s.processMessageFailures), "count")
message.NewInt64Field(msg, "ProcessMessageSamples", s.processMessageSamples, "count")
var tmp int64 = 0
if s.processMessageSamples > 0 {
tmp = s.processMessageDuration / s.processMessageSamples
}
message.NewInt64Field(msg, "ProcessMessageAvgDuration", tmp, "ns")
return nil
}
// Adds running filters count to the report output.
func (this *SandboxManagerFilter) ReportMsg(msg *message.Message) error {
message.NewIntField(msg, "RunningFilters", int(atomic.LoadInt32(&this.currentFilters)),
"count")
message.NewInt64Field(msg, "ProcessMessageCount",
atomic.LoadInt64(&this.processMessageCount), "count")
return nil
}
// Given a PluginRunner and a Message struct, this function will populate the
// Message struct's field values with the plugin's input channel length and
// capacity, plus any additional data that the plugin might provide through
// implementation of the `ReportingPlugin` interface defined above.
func PopulateReportMsg(pr PluginRunner, msg *message.Message) (err error) {
if reporter, ok := pr.Plugin().(ReportingPlugin); ok {
if err = reporter.ReportMsg(msg); err != nil {
return
}
}
if fRunner, ok := pr.(FilterRunner); ok {
message.NewIntField(msg, "InChanCapacity", cap(fRunner.InChan()), "count")
message.NewIntField(msg, "InChanLength", len(fRunner.InChan()), "count")
message.NewIntField(msg, "MatchChanCapacity", cap(fRunner.MatchRunner().inChan), "count")
message.NewIntField(msg, "MatchChanLength", len(fRunner.MatchRunner().inChan), "count")
message.NewIntField(msg, "LeakCount", fRunner.LeakCount(), "count")
var tmp int64 = 0
fRunner.MatchRunner().reportLock.Lock()
if fRunner.MatchRunner().matchSamples > 0 {
tmp = fRunner.MatchRunner().matchDuration / fRunner.MatchRunner().matchSamples
}
fRunner.MatchRunner().reportLock.Unlock()
message.NewInt64Field(msg, "MatchAvgDuration", tmp, "ns")
} else if dRunner, ok := pr.(DecoderRunner); ok {
message.NewIntField(msg, "InChanCapacity", cap(dRunner.InChan()), "count")
message.NewIntField(msg, "InChanLength", len(dRunner.InChan()), "count")
}
msg.SetType("heka.plugin-report")
return
}
// Satisfies the `pipeline.ReportingPlugin` interface to provide sandbox state
// information to the Heka report and dashboard.
func (this *SandboxFilter) ReportMsg(msg *message.Message) error {
this.reportLock.Lock()
defer this.reportLock.Unlock()
if this.sb == nil { // Plugin not initialized or running
return nil
}
message.NewIntField(msg, "Memory", int(this.sb.Usage(TYPE_MEMORY,
STAT_CURRENT)), "B")
message.NewIntField(msg, "MaxMemory", int(this.sb.Usage(TYPE_MEMORY,
STAT_MAXIMUM)), "B")
message.NewIntField(msg, "MaxInstructions", int(this.sb.Usage(
TYPE_INSTRUCTIONS, STAT_MAXIMUM)), "count")
message.NewIntField(msg, "MaxOutput", int(this.sb.Usage(TYPE_OUTPUT,
STAT_MAXIMUM)), "B")
message.NewInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&this.processMessageCount), "count")
message.NewInt64Field(msg, "ProcessMessageFailures", atomic.LoadInt64(&this.processMessageFailures), "count")
message.NewInt64Field(msg, "InjectMessageCount", atomic.LoadInt64(&this.injectMessageCount), "count")
message.NewInt64Field(msg, "ProcessMessageSamples", this.processMessageSamples, "count")
message.NewInt64Field(msg, "TimerEventSamples", this.timerEventSamples, "count")
var tmp int64 = 0
if this.processMessageSamples > 0 {
tmp = this.processMessageDuration / this.processMessageSamples
}
message.NewInt64Field(msg, "ProcessMessageAvgDuration", tmp, "ns")
tmp = 0
if this.profileMessageSamples > 0 {
message.NewInt64Field(msg, "ProfileMessageSamples", this.profileMessageSamples, "count")
tmp = this.profileMessageDuration / this.profileMessageSamples
message.NewInt64Field(msg, "ProfileMessageAvgDuration", tmp, "ns")
}
tmp = 0
if this.timerEventSamples > 0 {
tmp = this.timerEventDuration / this.timerEventSamples
}
message.NewInt64Field(msg, "TimerEventAvgDuration", tmp, "ns")
return nil
}
// Satisfies the `pipeline.ReportingPlugin` interface to provide sandbox state
// information to the Heka report and dashboard.
func (s *SandboxInput) ReportMsg(msg *message.Message) error {
s.reportLock.Lock()
defer s.reportLock.Unlock()
if s.sb == nil {
return fmt.Errorf("Input is not running")
}
message.NewIntField(msg, "Memory", int(s.sb.Usage(TYPE_MEMORY,
STAT_CURRENT)), "B")
message.NewIntField(msg, "MaxMemory", int(s.sb.Usage(TYPE_MEMORY,
STAT_MAXIMUM)), "B")
message.NewIntField(msg, "MaxInstructions", int(s.sb.Usage(
TYPE_INSTRUCTIONS, STAT_MAXIMUM)), "count")
message.NewIntField(msg, "MaxOutput", int(s.sb.Usage(TYPE_OUTPUT,
STAT_MAXIMUM)), "B")
message.NewInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&s.processMessageCount), "count")
message.NewInt64Field(msg, "ProcessMessageFailures", atomic.LoadInt64(&s.processMessageFailures), "count")
message.NewInt64Field(msg, "ProcessMessageBytes", atomic.LoadInt64(&s.processMessageBytes), "B")
return nil
}
// Generate recycle channel and plugin report messages and put them on the
// provided channel as they're ready.
func (pc *PipelineConfig) reports(reportChan chan *PipelinePack) {
var (
f *message.Field
pack *PipelinePack
msg *message.Message
err, e error
)
pack = <-pc.reportRecycleChan
msg = pack.Message
message.NewIntField(msg, "InChanCapacity", cap(pc.inputRecycleChan), "count")
message.NewIntField(msg, "InChanLength", len(pc.inputRecycleChan), "count")
msg.SetType("heka.input-report")
message.NewStringField(msg, "name", "inputRecycleChan")
message.NewStringField(msg, "key", "globals")
reportChan <- pack
pack = <-pc.reportRecycleChan
msg = pack.Message
message.NewIntField(msg, "InChanCapacity", cap(pc.injectRecycleChan), "count")
message.NewIntField(msg, "InChanLength", len(pc.injectRecycleChan), "count")
msg.SetType("heka.inject-report")
message.NewStringField(msg, "name", "injectRecycleChan")
message.NewStringField(msg, "key", "globals")
reportChan <- pack
pack = <-pc.reportRecycleChan
msg = pack.Message
message.NewIntField(msg, "InChanCapacity", cap(pc.router.InChan()), "count")
message.NewIntField(msg, "InChanLength", len(pc.router.InChan()), "count")
message.NewInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&pc.router.processMessageCount), "count")
msg.SetType("heka.router-report")
message.NewStringField(msg, "name", "Router")
message.NewStringField(msg, "key", "globals")
reportChan <- pack
getReport := func(runner PluginRunner) (pack *PipelinePack) {
pack = <-pc.reportRecycleChan
if err = PopulateReportMsg(runner, pack.Message); err != nil {
msg = pack.Message
f, e = message.NewField("Error", err.Error(), "")
if e == nil {
msg.AddField(f)
}
msg.SetType("heka.plugin-report")
}
return
}
pc.inputsLock.Lock()
for name, runner := range pc.InputRunners {
if runner.Transient() {
continue
}
pack = getReport(runner)
message.NewStringField(pack.Message, "name", name)
message.NewStringField(pack.Message, "key", "inputs")
reportChan <- pack
}
pc.inputsLock.Unlock()
for _, runner := range pc.allDecoders {
pack = getReport(runner)
message.NewStringField(pack.Message, "name", runner.Name())
message.NewStringField(pack.Message, "key", "decoders")
reportChan <- pack
}
pc.filtersLock.Lock()
for name, runner := range pc.FilterRunners {
pack = getReport(runner)
message.NewStringField(pack.Message, "name", name)
message.NewStringField(pack.Message, "key", "filters")
reportChan <- pack
}
pc.filtersLock.Unlock()
for name, runner := range pc.OutputRunners {
pack = getReport(runner)
message.NewStringField(pack.Message, "name", name)
message.NewStringField(pack.Message, "key", "outputs")
reportChan <- pack
}
close(reportChan)
}
func (this *SandboxFilter) Run(fr pipeline.FilterRunner, h pipeline.PluginHelper) (err error) {
inChan := fr.InChan()
ticker := fr.Ticker()
var (
ok = true
terminated = false
sample = true
blocking = false
backpressure = false
pack *pipeline.PipelinePack
retval int
msgLoopCount uint
injectionCount uint
startTime time.Time
slowDuration int64 = int64(this.pConfig.Globals.MaxMsgProcessDuration)
duration int64
capacity = cap(inChan) - 1
)
// We assign to the return value of Run() for errors in the closure so that
// the plugin runner can determine what caused the SandboxFilter to return.
this.sb.InjectMessage(func(payload, payload_type, payload_name string) int {
if injectionCount == 0 {
err = pipeline.TerminatedError("exceeded InjectMessage count")
return 2
}
injectionCount--
pack := h.PipelinePack(msgLoopCount)
if pack == nil {
err = pipeline.TerminatedError(fmt.Sprintf("exceeded MaxMsgLoops = %d",
this.pConfig.Globals.MaxMsgLoops))
return 3
}
if len(payload_type) == 0 { // heka protobuf message
hostname := pack.Message.GetHostname()
err := proto.Unmarshal([]byte(payload), pack.Message)
if err == nil {
// do not allow filters to override the following
pack.Message.SetType("heka.sandbox." + pack.Message.GetType())
pack.Message.SetLogger(fr.Name())
pack.Message.SetHostname(hostname)
} else {
return 1
}
} else {
pack.Message.SetType("heka.sandbox-output")
pack.Message.SetLogger(fr.Name())
pack.Message.SetPayload(payload)
ptype, _ := message.NewField("payload_type", payload_type, "file-extension")
pack.Message.AddField(ptype)
pname, _ := message.NewField("payload_name", payload_name, "")
pack.Message.AddField(pname)
}
if !fr.Inject(pack) {
return 4
}
atomic.AddInt64(&this.injectMessageCount, 1)
return 0
})
for ok {
select {
case pack, ok = <-inChan:
if !ok {
break
}
atomic.AddInt64(&this.processMessageCount, 1)
injectionCount = this.pConfig.Globals.MaxMsgProcessInject
msgLoopCount = pack.MsgLoopCount
if this.manager != nil { // only check for backpressure on dynamic plugins
// reading a channel length is generally fast ~1ns
// we need to check the entire chain back to the router
backpressure = len(inChan) >= capacity ||
fr.MatchRunner().InChanLen() >= capacity ||
len(h.PipelineConfig().Router().InChan()) >= capacity
}
// performing the timing is expensive ~40ns but if we are
// backpressured we need a decent sample set before triggering
// termination
if sample ||
(backpressure && this.processMessageSamples < int64(capacity)) ||
this.sbc.Profile {
startTime = time.Now()
sample = true
}
retval = this.sb.ProcessMessage(pack)
if sample {
duration = time.Since(startTime).Nanoseconds()
this.reportLock.Lock()
this.processMessageDuration += duration
this.processMessageSamples++
if this.sbc.Profile {
this.profileMessageDuration = this.processMessageDuration
this.profileMessageSamples = this.processMessageSamples
if this.profileMessageSamples == int64(capacity)*10 {
this.sbc.Profile = false
// reset the normal sampling so it isn't heavily skewed by the profile values
// i.e. process messages fast during profiling and then switch to malicious code
this.processMessageDuration = this.profileMessageDuration / this.profileMessageSamples
this.processMessageSamples = 1
}
}
this.reportLock.Unlock()
}
if retval <= 0 {
if backpressure && this.processMessageSamples >= int64(capacity) {
if this.processMessageDuration/this.processMessageSamples > slowDuration ||
fr.MatchRunner().GetAvgDuration() > slowDuration/5 {
terminated = true
blocking = true
}
}
if retval < 0 {
atomic.AddInt64(&this.processMessageFailures, 1)
em := this.sb.LastError()
if len(em) > 0 {
fr.LogError(errors.New(em))
}
}
sample = 0 == rand.Intn(this.sampleDenominator)
} else {
terminated = true
}
pack.Recycle()
case t := <-ticker:
injectionCount = this.pConfig.Globals.MaxMsgTimerInject
startTime = time.Now()
if retval = this.sb.TimerEvent(t.UnixNano()); retval != 0 {
terminated = true
}
duration = time.Since(startTime).Nanoseconds()
this.reportLock.Lock()
this.timerEventDuration += duration
this.timerEventSamples++
this.reportLock.Unlock()
}
if terminated {
pack := h.PipelinePack(0)
pack.Message.SetType("heka.sandbox-terminated")
pack.Message.SetLogger(pipeline.HEKA_DAEMON)
message.NewStringField(pack.Message, "plugin", fr.Name())
if blocking {
pack.Message.SetPayload("sandbox is running slowly and blocking the router")
// no lock on the ProcessMessage variables here because there are no active writers
message.NewInt64Field(pack.Message, "ProcessMessageCount", this.processMessageCount, "count")
message.NewInt64Field(pack.Message, "ProcessMessageFailures", this.processMessageFailures, "count")
message.NewInt64Field(pack.Message, "ProcessMessageSamples", this.processMessageSamples, "count")
message.NewInt64Field(pack.Message, "ProcessMessageAvgDuration",
this.processMessageDuration/this.processMessageSamples, "ns")
message.NewInt64Field(pack.Message, "MatchAvgDuration", fr.MatchRunner().GetAvgDuration(), "ns")
message.NewIntField(pack.Message, "FilterChanLength", len(inChan), "count")
message.NewIntField(pack.Message, "MatchChanLength", fr.MatchRunner().InChanLen(), "count")
message.NewIntField(pack.Message, "RouterChanLength", len(h.PipelineConfig().Router().InChan()), "count")
} else {
pack.Message.SetPayload(this.sb.LastError())
}
fr.Inject(pack)
break
}
}
if this.manager != nil {
this.manager.PluginExited()
}
this.reportLock.Lock()
var destroyErr error
if this.sbc.PreserveData {
destroyErr = this.sb.Destroy(this.preservationFile)
} else {
destroyErr = this.sb.Destroy("")
}
if destroyErr != nil {
err = destroyErr
}
this.sb = nil
this.reportLock.Unlock()
return
}
func TestGraphiteHelpers(t *testing.T) {
var sbc SandboxConfig
sbc.ScriptFilename = "./testsupport/graphite.lua"
sbc.ModuleDirectory = "./modules"
sbc.MemoryLimit = 100000
sbc.InstructionLimit = 1000
sbc.OutputLimit = 8000
sbc.Config = make(map[string]interface{})
sb, err := lua.CreateLuaSandbox(&sbc)
if err != nil {
t.Errorf("%s", err)
}
err = sb.Init("")
if err != nil {
t.Errorf("%s", err)
}
for i := 0; i < 4; i++ {
pack := getTestPack()
pack.Message.SetHostname("localhost")
pack.Message.SetLogger("GoSpec")
message.NewIntField(pack.Message, "status", 200, "status")
message.NewIntField(pack.Message, "request_time", 15*i, "request_time")
r := sb.ProcessMessage(pack)
if r != 0 {
t.Errorf("Graphite returned %s", r)
}
}
injectCount := 0
sb.InjectMessage(func(payload, payload_type, payload_name string) int {
graphite_payload := `stats.counters.localhost.nginx.GoSpec.http_200.count 4 0
stats.counters.localhost.nginx.GoSpec.http_200.rate 0.400000 0
stats.timers.localhost.nginx.GoSpec.request_time.count 4 0
stats.timers.localhost.nginx.GoSpec.request_time.count_ps 0.400000 0
stats.timers.localhost.nginx.GoSpec.request_time.lower 0.000000 0
stats.timers.localhost.nginx.GoSpec.request_time.upper 45.000000 0
stats.timers.localhost.nginx.GoSpec.request_time.sum 90.000000 0
stats.timers.localhost.nginx.GoSpec.request_time.mean 22.500000 0
stats.timers.localhost.nginx.GoSpec.request_time.mean_90 22.500000 0
stats.timers.localhost.nginx.GoSpec.request_time.upper_90 45.000000 0
stats.statsd.numStats 2 0
`
if payload_type != "txt" {
t.Errorf("Received payload type: %s", payload_type)
}
if payload_name != "statmetric" {
t.Errorf("Received payload name: %s", payload_name)
}
if graphite_payload != payload {
t.Errorf("Received payload: %s", payload)
}
injectCount += 1
return 0
})
sb.TimerEvent(200)
if injectCount > 0 {
t.Errorf("Looks there was an error during timer_event")
}
sb.Destroy("")
}