// Convenience function for creating and setting a string field called "name"
// on a message object.
func newStringField(msg *message.Message, name string, val string) {
if f, err := message.NewField(name, val, ""); err == nil {
msg.AddField(f)
} else {
fmt.Println("Report error adding string field: ", err)
}
}
func PopulateReportMsg(pr PluginRunner, msg *message.Message) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("'%s' `populateReportMsg` panic: %s", pr.Name(), r)
}
}()
if reporter, ok := pr.Plugin().(ReportingPlugin); ok {
if err = reporter.ReportMsg(msg); err != nil {
return
}
}
if fRunner, ok := pr.(FilterRunner); ok {
newIntField(msg, "InChanCapacity", cap(fRunner.InChan()))
newIntField(msg, "InChanLength", len(fRunner.InChan()))
} else if dRunner, ok := pr.(DecoderRunner); ok {
newIntField(msg, "InChanCapacity", cap(dRunner.InChan()))
newIntField(msg, "InChanLength", len(dRunner.InChan()))
}
if msg.GetType() != "" {
var f *message.Field
f, err = message.NewField("Type", msg.GetType(), message.Field_RAW)
if err != nil {
return
}
msg.AddField(f)
}
msg.SetType("heka.plugin-report")
return
}
// Convenience function for creating a new int64 field on a message object.
func newInt64Field(msg *message.Message, name string, val int64, representation string) {
if f, err := message.NewField(name, val, representation); err == nil {
msg.AddField(f)
} else {
fmt.Println("Report error adding int64 field: ", err)
}
}
// Applies this message template's values to the provided message object,
// interpolating the provided substitutions into the values in the process.
func (mt MessageTemplate) PopulateMessage(msg *message.Message, subs map[string]string) error {
var val string
for field, rawVal := range mt {
val = InterpolateString(rawVal, subs)
switch field {
case "Logger":
msg.SetLogger(val)
case "Type":
msg.SetType(val)
case "Payload":
msg.SetPayload(val)
case "Hostname":
msg.SetHostname(val)
case "Pid":
pid, err := strconv.ParseInt(val, 10, 32)
if err != nil {
return err
}
msg.SetPid(int32(pid))
case "Uuid":
msg.SetUuid([]byte(val))
default:
fi := strings.SplitN(field, "|", 2)
if len(fi) < 2 {
fi = append(fi, "")
}
f, err := message.NewField(fi[0], val, fi[1])
msg.AddField(f)
if err != nil {
return err
}
}
}
return nil
}
// Applies this message template's values to the provided message object,
// interpolating the provided substitutions into the values in the process.
func (mt MessageTemplate) PopulateMessage(msg *message.Message, subs map[string]string) error {
var val string
for field, rawVal := range mt {
if subs == nil {
val = rawVal
} else {
val = InterpolateString(rawVal, subs)
}
switch field {
case "Logger":
msg.SetLogger(val)
case "Type":
msg.SetType(val)
case "Payload":
msg.SetPayload(val)
case "Hostname":
msg.SetHostname(val)
case "Pid":
intPart := strings.Split(val, ".")[0]
pid, err := strconv.ParseInt(intPart, 10, 32)
if err != nil {
return err
}
msg.SetPid(int32(pid))
case "Severity":
severity, err := strconv.ParseInt(val, 10, 32)
if err != nil {
return err
}
msg.SetSeverity(int32(severity))
case "Uuid":
if len(val) == message.UUID_SIZE {
msg.SetUuid([]byte(val))
} else {
if uuidBytes := uuid.Parse(val); uuidBytes == nil {
return errors.New("Invalid UUID string.")
} else {
msg.SetUuid(uuidBytes)
}
}
default:
fi := strings.SplitN(field, "|", 2)
if len(fi) < 2 {
fi = append(fi, "")
}
f, err := message.NewField(fi[0], val, fi[1])
msg.AddField(f)
if err != nil {
return err
}
}
}
return nil
}
// Fields are additional logging data passed to Heka. They are technically
// undefined, but searchable and actionable.
func addFields(msg *message.Message, fields Fields) (err error) {
for key, ival := range fields {
var field *message.Field
if ival == "" {
ival = "*empty*"
}
if key == "" {
continue
}
field, err = message.NewField(key, ival, ival)
if err != nil {
return err
}
msg.AddField(field)
}
return err
}
// AddDecodeFailureFields adds two fields to the provided message object. The
// first field is a boolean field called `decode_failure`, set to true. The
// second is a string field called `decode_error` which will contain the
// provided error message, truncated to 500 bytes if necessary.
func AddDecodeFailureFields(m *message.Message, errMsg string) error {
field0, err := message.NewField("decode_failure", true, "")
if err != nil {
err = fmt.Errorf("field creation error: %s", err.Error())
return err
}
if len(errMsg) > 500 {
errMsg = errMsg[:500]
}
field1, err := message.NewField("decode_error", errMsg, "")
if err != nil {
err = fmt.Errorf("field creation error: %s", err.Error())
return err
}
m.AddField(field0)
m.AddField(field1)
return nil
}
func (i *StatAccumInput) ReportMsg(msg *message.Message) (err error) {
msg.AddField(f0)
msg.AddField(f1)
return
}
func (f *CounterFilter) ReportMsg(msg *message.Message) (err error) {
msg.AddField(f0)
msg.AddField(f1)
return
}
// Convenience function for creating and setting a string field called "name"
// on a message object.
func setNameField(msg *message.Message, name string) {
f, err := message.NewField("name", name, message.Field_RAW)
if err == nil {
msg.AddField(f)
}
}
// Convenience function for creating a new integer field on a message object.
func newIntField(msg *message.Message, name string, val int) {
f, err := message.NewField(name, val, message.Field_RAW)
if err == nil {
msg.AddField(f)
}
}
func addField(name string, value interface{}, msg *message.Message) {
field, err := message.NewField(name, value, "")
if err == nil {
msg.AddField(field)
}
}
// function which fills all part of Heka message
func setHekaMessageFields(m plugin.MetricType, msg *message.Message) error {
mName := make([]string, 0, len(m.Namespace()))
var dimField *message.Field
var err error
// Loop on namespace elements
for _, elt := range m.Namespace() {
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Namespace %#+v",
elt))
// Dynamic element is not inserted in metric name
// but rather added to dimension field
if elt.IsDynamic() {
dimField, err = addToDimensions(dimField, elt.Name)
if err != nil {
logger.WithField("_block", "setHekaMessageFields").Error(err)
return err
}
addField(elt.Name, elt.Value, msg)
} else {
// Static element is concatenated to metric name
mName = append(mName, elt.Value)
}
}
// Processing of tags
if len(m.Tags()) > 0 {
for tag, value := range m.Tags() {
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Adding tag=%s value=%s",
tag, value))
dimField, err = addToDimensions(dimField, tag)
if err != nil {
logger.WithField("_block", "setHekaMessageFields").Error(err)
return err
}
addField(tag, value, msg)
}
}
if dimField != nil {
msg.AddField(dimField)
}
// Handle metric name
metricName := strings.Join(mName, ".")
// TODO protect access using mutex
// for potential race conditions
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Checking metric=%s",
metricName))
// Is mapping already stored
if val, ok := MetricMappings[metricName]; ok {
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Metric=%s in cache %s",
metricName, val))
metricName = val
} else {
oldMetricName := metricName
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Metric=%s not in cache",
metricName))
// Namespace handling
for kmapping, vmapping := range globalMappings.Namespace {
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Checking metric=%s against namespace %s (%s)",
metricName, kmapping, vmapping))
// Try to see if substitution changes something
newMetricName := strings.Replace(metricName, kmapping, vmapping, 1)
if strings.Compare(newMetricName, metricName) != 0 {
MetricMappings[oldMetricName] = newMetricName
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Changing metric=%s into %s",
metricName, newMetricName))
metricName = newMetricName
}
}
// Metrics handling
for kmapping, vmapping := range globalMappings.Metrics {
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Checking metric=%s against metric %s (%s)",
metricName, kmapping, vmapping))
// Try to see if substitution changes something
newMetricName := strings.Replace(metricName, kmapping, vmapping, 1)
if strings.Compare(newMetricName, metricName) != 0 {
MetricMappings[oldMetricName] = newMetricName
logger.WithField("_block", "setHekaMessageFields").Debug(
fmt.Sprintf("Changing metric=%s into %s",
metricName, newMetricName))
metricName = newMetricName
}
}
}
addField("name", metricName, msg)
addField("value", getData(m.Data()), msg)
addField("timestamp", m.Timestamp().UnixNano(), msg)
return nil
}
// Convenience function for creating a new int64 field on a message object.
func newInt64Field(msg *message.Message, name string, val int64, representation string) {
f, err := message.NewField(name, val, representation)
if err == nil {
msg.AddField(f)
}
}
// Generate recycle channel and plugin report messages and put them on the
// provided channel as they're ready.
func (pc *PipelineConfig) reports(reportChan chan *PipelinePack) {
var (
f *message.Field
pack *PipelinePack
msg *message.Message
err, e error
)
pack = pc.PipelinePack(0)
msg = pack.Message
newIntField(msg, "InChanCapacity", cap(pc.inputRecycleChan), "count")
newIntField(msg, "InChanLength", len(pc.inputRecycleChan), "count")
msg.SetType("heka.input-report")
setNameField(msg, "inputRecycleChan")
reportChan <- pack
pack = pc.PipelinePack(0)
msg = pack.Message
newIntField(msg, "InChanCapacity", cap(pc.injectRecycleChan), "count")
newIntField(msg, "InChanLength", len(pc.injectRecycleChan), "count")
msg.SetType("heka.inject-report")
setNameField(msg, "injectRecycleChan")
reportChan <- pack
pack = pc.PipelinePack(0)
msg = pack.Message
newIntField(msg, "InChanCapacity", cap(pc.router.InChan()), "count")
newIntField(msg, "InChanLength", len(pc.router.InChan()), "count")
newInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&pc.router.processMessageCount), "count")
msg.SetType("heka.router-report")
setNameField(msg, "Router")
reportChan <- pack
getReport := func(runner PluginRunner) (pack *PipelinePack) {
pack = pc.PipelinePack(0)
if err = PopulateReportMsg(runner, pack.Message); err != nil {
msg = pack.Message
f, e = message.NewField("Error", err.Error(), "")
if e == nil {
msg.AddField(f)
}
msg.SetType("heka.plugin-report")
}
return
}
for name, runner := range pc.InputRunners {
pack = getReport(runner)
setNameField(pack.Message, name)
reportChan <- pack
}
for _, runner := range pc.allDecoders {
pack = getReport(runner)
setNameField(pack.Message, runner.Name())
reportChan <- pack
}
for name, dChan := range pc.decoderChannels {
pack = pc.PipelinePack(0)
msg = pack.Message
msg.SetType("heka.decoder-pool-report")
setNameField(msg, fmt.Sprintf("DecoderPool-%s", name))
newIntField(msg, "InChanCapacity", cap(dChan), "count")
newIntField(msg, "InChanLength", len(dChan), "count")
reportChan <- pack
}
for name, runner := range pc.FilterRunners {
pack = getReport(runner)
setNameField(pack.Message, name)
reportChan <- pack
}
for name, runner := range pc.OutputRunners {
pack = getReport(runner)
setNameField(pack.Message, name)
reportChan <- pack
}
close(reportChan)
}
// Generate recycle channel and plugin report messages and put them on the
// provided channel as they're ready.
func (pc *PipelineConfig) reports(reportChan chan *PipelinePack) {
var (
f *message.Field
pack *PipelinePack
msg *message.Message
err, e error
)
pack = pc.PipelinePack(0)
msg = pack.Message
newIntField(msg, "InChanCapacity", cap(pc.inputRecycleChan))
newIntField(msg, "InChanLength", len(pc.inputRecycleChan))
msg.SetType("heka.input-report")
setNameField(msg, "inputRecycleChan")
reportChan <- pack
pack = pc.PipelinePack(0)
msg = pack.Message
newIntField(msg, "InChanCapacity", cap(pc.injectRecycleChan))
newIntField(msg, "InChanLength", len(pc.injectRecycleChan))
msg.SetType("heka.inject-report")
setNameField(msg, "injectRecycleChan")
reportChan <- pack
pack = pc.PipelinePack(0)
msg = pack.Message
newIntField(msg, "InChanCapacity", cap(pc.router.InChan()))
newIntField(msg, "InChanLength", len(pc.router.InChan()))
msg.SetType("heka.router-report")
setNameField(msg, "Router")
reportChan <- pack
getReport := func(runner PluginRunner) (pack *PipelinePack) {
pack = pc.PipelinePack(0)
if err = PopulateReportMsg(runner, pack.Message); err != nil {
msg = pack.Message
f, e = message.NewField("Error", err.Error(), message.Field_RAW)
if e == nil {
msg.AddField(f)
}
msg.SetType("heka.plugin-report")
}
return
}
for name, runner := range pc.InputRunners {
pack = getReport(runner)
if len(pack.Message.Fields) > 0 || pack.Message.GetPayload() != "" {
setNameField(pack.Message, name)
reportChan <- pack
} else {
pack.Recycle()
}
}
for i, dSet := range pc.DecoderSets {
for name, runner := range dSet.AllByName() {
pack = getReport(runner)
setNameField(pack.Message, fmt.Sprintf("%s-%d", name, i))
reportChan <- pack
}
}
for name, runner := range pc.FilterRunners {
pack = getReport(runner)
setNameField(pack.Message, name)
reportChan <- pack
}
for name, runner := range pc.OutputRunners {
pack = getReport(runner)
setNameField(pack.Message, name)
reportChan <- pack
}
close(reportChan)
}
// Generate recycle channel and plugin report messages and put them on the
// provided channel as they're ready.
func (pc *PipelineConfig) reports(reportChan chan *PipelinePack) {
var (
f *message.Field
pack *PipelinePack
msg *message.Message
err, e error
)
pack = <-pc.reportRecycleChan
msg = pack.Message
message.NewIntField(msg, "InChanCapacity", cap(pc.inputRecycleChan), "count")
message.NewIntField(msg, "InChanLength", len(pc.inputRecycleChan), "count")
msg.SetType("heka.input-report")
message.NewStringField(msg, "name", "inputRecycleChan")
message.NewStringField(msg, "key", "globals")
reportChan <- pack
pack = <-pc.reportRecycleChan
msg = pack.Message
message.NewIntField(msg, "InChanCapacity", cap(pc.injectRecycleChan), "count")
message.NewIntField(msg, "InChanLength", len(pc.injectRecycleChan), "count")
msg.SetType("heka.inject-report")
message.NewStringField(msg, "name", "injectRecycleChan")
message.NewStringField(msg, "key", "globals")
reportChan <- pack
pack = <-pc.reportRecycleChan
msg = pack.Message
message.NewIntField(msg, "InChanCapacity", cap(pc.router.InChan()), "count")
message.NewIntField(msg, "InChanLength", len(pc.router.InChan()), "count")
message.NewInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&pc.router.processMessageCount), "count")
msg.SetType("heka.router-report")
message.NewStringField(msg, "name", "Router")
message.NewStringField(msg, "key", "globals")
reportChan <- pack
getReport := func(runner PluginRunner) (pack *PipelinePack) {
pack = <-pc.reportRecycleChan
if err = PopulateReportMsg(runner, pack.Message); err != nil {
msg = pack.Message
f, e = message.NewField("Error", err.Error(), "")
if e == nil {
msg.AddField(f)
}
msg.SetType("heka.plugin-report")
}
return
}
pc.inputsLock.Lock()
for name, runner := range pc.InputRunners {
if runner.Transient() {
continue
}
pack = getReport(runner)
message.NewStringField(pack.Message, "name", name)
message.NewStringField(pack.Message, "key", "inputs")
reportChan <- pack
}
pc.inputsLock.Unlock()
for _, runner := range pc.allDecoders {
pack = getReport(runner)
message.NewStringField(pack.Message, "name", runner.Name())
message.NewStringField(pack.Message, "key", "decoders")
reportChan <- pack
}
pc.filtersLock.Lock()
for name, runner := range pc.FilterRunners {
pack = getReport(runner)
message.NewStringField(pack.Message, "name", name)
message.NewStringField(pack.Message, "key", "filters")
reportChan <- pack
}
pc.filtersLock.Unlock()
for name, runner := range pc.OutputRunners {
pack = getReport(runner)
message.NewStringField(pack.Message, "name", name)
message.NewStringField(pack.Message, "key", "outputs")
reportChan <- pack
}
close(reportChan)
}
func add_field(msg *message.Message, field_name string, value interface{}) {
f, _ := message.NewField(field_name, value, message.Field_RAW)
msg.AddField(f)
}