// Convenience function for creating and setting a string field called "name" // on a message object. func newStringField(msg *message.Message, name string, val string) { if f, err := message.NewField(name, val, ""); err == nil { msg.AddField(f) } else { fmt.Println("Report error adding string field: ", err) } }
func PopulateReportMsg(pr PluginRunner, msg *message.Message) (err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("'%s' `populateReportMsg` panic: %s", pr.Name(), r) } }() if reporter, ok := pr.Plugin().(ReportingPlugin); ok { if err = reporter.ReportMsg(msg); err != nil { return } } if fRunner, ok := pr.(FilterRunner); ok { newIntField(msg, "InChanCapacity", cap(fRunner.InChan())) newIntField(msg, "InChanLength", len(fRunner.InChan())) } else if dRunner, ok := pr.(DecoderRunner); ok { newIntField(msg, "InChanCapacity", cap(dRunner.InChan())) newIntField(msg, "InChanLength", len(dRunner.InChan())) } if msg.GetType() != "" { var f *message.Field f, err = message.NewField("Type", msg.GetType(), message.Field_RAW) if err != nil { return } msg.AddField(f) } msg.SetType("heka.plugin-report") return }
// Convenience function for creating a new int64 field on a message object. func newInt64Field(msg *message.Message, name string, val int64, representation string) { if f, err := message.NewField(name, val, representation); err == nil { msg.AddField(f) } else { fmt.Println("Report error adding int64 field: ", err) } }
// Applies this message template's values to the provided message object, // interpolating the provided substitutions into the values in the process. func (mt MessageTemplate) PopulateMessage(msg *message.Message, subs map[string]string) error { var val string for field, rawVal := range mt { val = InterpolateString(rawVal, subs) switch field { case "Logger": msg.SetLogger(val) case "Type": msg.SetType(val) case "Payload": msg.SetPayload(val) case "Hostname": msg.SetHostname(val) case "Pid": pid, err := strconv.ParseInt(val, 10, 32) if err != nil { return err } msg.SetPid(int32(pid)) case "Uuid": msg.SetUuid([]byte(val)) default: fi := strings.SplitN(field, "|", 2) if len(fi) < 2 { fi = append(fi, "") } f, err := message.NewField(fi[0], val, fi[1]) msg.AddField(f) if err != nil { return err } } } return nil }
// Applies this message template's values to the provided message object, // interpolating the provided substitutions into the values in the process. func (mt MessageTemplate) PopulateMessage(msg *message.Message, subs map[string]string) error { var val string for field, rawVal := range mt { if subs == nil { val = rawVal } else { val = InterpolateString(rawVal, subs) } switch field { case "Logger": msg.SetLogger(val) case "Type": msg.SetType(val) case "Payload": msg.SetPayload(val) case "Hostname": msg.SetHostname(val) case "Pid": intPart := strings.Split(val, ".")[0] pid, err := strconv.ParseInt(intPart, 10, 32) if err != nil { return err } msg.SetPid(int32(pid)) case "Severity": severity, err := strconv.ParseInt(val, 10, 32) if err != nil { return err } msg.SetSeverity(int32(severity)) case "Uuid": if len(val) == message.UUID_SIZE { msg.SetUuid([]byte(val)) } else { if uuidBytes := uuid.Parse(val); uuidBytes == nil { return errors.New("Invalid UUID string.") } else { msg.SetUuid(uuidBytes) } } default: fi := strings.SplitN(field, "|", 2) if len(fi) < 2 { fi = append(fi, "") } f, err := message.NewField(fi[0], val, fi[1]) msg.AddField(f) if err != nil { return err } } } return nil }
// Fields are additional logging data passed to Heka. They are technically // undefined, but searchable and actionable. func addFields(msg *message.Message, fields Fields) (err error) { for key, ival := range fields { var field *message.Field if ival == "" { ival = "*empty*" } if key == "" { continue } field, err = message.NewField(key, ival, ival) if err != nil { return err } msg.AddField(field) } return err }
// AddDecodeFailureFields adds two fields to the provided message object. The // first field is a boolean field called `decode_failure`, set to true. The // second is a string field called `decode_error` which will contain the // provided error message, truncated to 500 bytes if necessary. func AddDecodeFailureFields(m *message.Message, errMsg string) error { field0, err := message.NewField("decode_failure", true, "") if err != nil { err = fmt.Errorf("field creation error: %s", err.Error()) return err } if len(errMsg) > 500 { errMsg = errMsg[:500] } field1, err := message.NewField("decode_error", errMsg, "") if err != nil { err = fmt.Errorf("field creation error: %s", err.Error()) return err } m.AddField(field0) m.AddField(field1) return nil }
func (i *StatAccumInput) ReportMsg(msg *message.Message) (err error) { msg.AddField(f0) msg.AddField(f1) return }
func (f *CounterFilter) ReportMsg(msg *message.Message) (err error) { msg.AddField(f0) msg.AddField(f1) return }
// Convenience function for creating and setting a string field called "name" // on a message object. func setNameField(msg *message.Message, name string) { f, err := message.NewField("name", name, message.Field_RAW) if err == nil { msg.AddField(f) } }
// Convenience function for creating a new integer field on a message object. func newIntField(msg *message.Message, name string, val int) { f, err := message.NewField(name, val, message.Field_RAW) if err == nil { msg.AddField(f) } }
func addField(name string, value interface{}, msg *message.Message) { field, err := message.NewField(name, value, "") if err == nil { msg.AddField(field) } }
// function which fills all part of Heka message func setHekaMessageFields(m plugin.MetricType, msg *message.Message) error { mName := make([]string, 0, len(m.Namespace())) var dimField *message.Field var err error // Loop on namespace elements for _, elt := range m.Namespace() { logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Namespace %#+v", elt)) // Dynamic element is not inserted in metric name // but rather added to dimension field if elt.IsDynamic() { dimField, err = addToDimensions(dimField, elt.Name) if err != nil { logger.WithField("_block", "setHekaMessageFields").Error(err) return err } addField(elt.Name, elt.Value, msg) } else { // Static element is concatenated to metric name mName = append(mName, elt.Value) } } // Processing of tags if len(m.Tags()) > 0 { for tag, value := range m.Tags() { logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Adding tag=%s value=%s", tag, value)) dimField, err = addToDimensions(dimField, tag) if err != nil { logger.WithField("_block", "setHekaMessageFields").Error(err) return err } addField(tag, value, msg) } } if dimField != nil { msg.AddField(dimField) } // Handle metric name metricName := strings.Join(mName, ".") // TODO protect access using mutex // for potential race conditions logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Checking metric=%s", metricName)) // Is mapping already stored if val, ok := MetricMappings[metricName]; ok { logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Metric=%s in cache %s", metricName, val)) metricName = val } else { oldMetricName := metricName logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Metric=%s not in cache", metricName)) // Namespace handling for kmapping, vmapping := range globalMappings.Namespace { logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Checking metric=%s against namespace %s (%s)", metricName, kmapping, vmapping)) // Try to see if substitution changes something newMetricName := strings.Replace(metricName, kmapping, vmapping, 1) if strings.Compare(newMetricName, metricName) != 0 { MetricMappings[oldMetricName] = newMetricName logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Changing metric=%s into %s", metricName, newMetricName)) metricName = newMetricName } } // Metrics handling for kmapping, vmapping := range globalMappings.Metrics { logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Checking metric=%s against metric %s (%s)", metricName, kmapping, vmapping)) // Try to see if substitution changes something newMetricName := strings.Replace(metricName, kmapping, vmapping, 1) if strings.Compare(newMetricName, metricName) != 0 { MetricMappings[oldMetricName] = newMetricName logger.WithField("_block", "setHekaMessageFields").Debug( fmt.Sprintf("Changing metric=%s into %s", metricName, newMetricName)) metricName = newMetricName } } } addField("name", metricName, msg) addField("value", getData(m.Data()), msg) addField("timestamp", m.Timestamp().UnixNano(), msg) return nil }
// Convenience function for creating a new int64 field on a message object. func newInt64Field(msg *message.Message, name string, val int64, representation string) { f, err := message.NewField(name, val, representation) if err == nil { msg.AddField(f) } }
// Generate recycle channel and plugin report messages and put them on the // provided channel as they're ready. func (pc *PipelineConfig) reports(reportChan chan *PipelinePack) { var ( f *message.Field pack *PipelinePack msg *message.Message err, e error ) pack = pc.PipelinePack(0) msg = pack.Message newIntField(msg, "InChanCapacity", cap(pc.inputRecycleChan), "count") newIntField(msg, "InChanLength", len(pc.inputRecycleChan), "count") msg.SetType("heka.input-report") setNameField(msg, "inputRecycleChan") reportChan <- pack pack = pc.PipelinePack(0) msg = pack.Message newIntField(msg, "InChanCapacity", cap(pc.injectRecycleChan), "count") newIntField(msg, "InChanLength", len(pc.injectRecycleChan), "count") msg.SetType("heka.inject-report") setNameField(msg, "injectRecycleChan") reportChan <- pack pack = pc.PipelinePack(0) msg = pack.Message newIntField(msg, "InChanCapacity", cap(pc.router.InChan()), "count") newIntField(msg, "InChanLength", len(pc.router.InChan()), "count") newInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&pc.router.processMessageCount), "count") msg.SetType("heka.router-report") setNameField(msg, "Router") reportChan <- pack getReport := func(runner PluginRunner) (pack *PipelinePack) { pack = pc.PipelinePack(0) if err = PopulateReportMsg(runner, pack.Message); err != nil { msg = pack.Message f, e = message.NewField("Error", err.Error(), "") if e == nil { msg.AddField(f) } msg.SetType("heka.plugin-report") } return } for name, runner := range pc.InputRunners { pack = getReport(runner) setNameField(pack.Message, name) reportChan <- pack } for _, runner := range pc.allDecoders { pack = getReport(runner) setNameField(pack.Message, runner.Name()) reportChan <- pack } for name, dChan := range pc.decoderChannels { pack = pc.PipelinePack(0) msg = pack.Message msg.SetType("heka.decoder-pool-report") setNameField(msg, fmt.Sprintf("DecoderPool-%s", name)) newIntField(msg, "InChanCapacity", cap(dChan), "count") newIntField(msg, "InChanLength", len(dChan), "count") reportChan <- pack } for name, runner := range pc.FilterRunners { pack = getReport(runner) setNameField(pack.Message, name) reportChan <- pack } for name, runner := range pc.OutputRunners { pack = getReport(runner) setNameField(pack.Message, name) reportChan <- pack } close(reportChan) }
// Generate recycle channel and plugin report messages and put them on the // provided channel as they're ready. func (pc *PipelineConfig) reports(reportChan chan *PipelinePack) { var ( f *message.Field pack *PipelinePack msg *message.Message err, e error ) pack = pc.PipelinePack(0) msg = pack.Message newIntField(msg, "InChanCapacity", cap(pc.inputRecycleChan)) newIntField(msg, "InChanLength", len(pc.inputRecycleChan)) msg.SetType("heka.input-report") setNameField(msg, "inputRecycleChan") reportChan <- pack pack = pc.PipelinePack(0) msg = pack.Message newIntField(msg, "InChanCapacity", cap(pc.injectRecycleChan)) newIntField(msg, "InChanLength", len(pc.injectRecycleChan)) msg.SetType("heka.inject-report") setNameField(msg, "injectRecycleChan") reportChan <- pack pack = pc.PipelinePack(0) msg = pack.Message newIntField(msg, "InChanCapacity", cap(pc.router.InChan())) newIntField(msg, "InChanLength", len(pc.router.InChan())) msg.SetType("heka.router-report") setNameField(msg, "Router") reportChan <- pack getReport := func(runner PluginRunner) (pack *PipelinePack) { pack = pc.PipelinePack(0) if err = PopulateReportMsg(runner, pack.Message); err != nil { msg = pack.Message f, e = message.NewField("Error", err.Error(), message.Field_RAW) if e == nil { msg.AddField(f) } msg.SetType("heka.plugin-report") } return } for name, runner := range pc.InputRunners { pack = getReport(runner) if len(pack.Message.Fields) > 0 || pack.Message.GetPayload() != "" { setNameField(pack.Message, name) reportChan <- pack } else { pack.Recycle() } } for i, dSet := range pc.DecoderSets { for name, runner := range dSet.AllByName() { pack = getReport(runner) setNameField(pack.Message, fmt.Sprintf("%s-%d", name, i)) reportChan <- pack } } for name, runner := range pc.FilterRunners { pack = getReport(runner) setNameField(pack.Message, name) reportChan <- pack } for name, runner := range pc.OutputRunners { pack = getReport(runner) setNameField(pack.Message, name) reportChan <- pack } close(reportChan) }
// Generate recycle channel and plugin report messages and put them on the // provided channel as they're ready. func (pc *PipelineConfig) reports(reportChan chan *PipelinePack) { var ( f *message.Field pack *PipelinePack msg *message.Message err, e error ) pack = <-pc.reportRecycleChan msg = pack.Message message.NewIntField(msg, "InChanCapacity", cap(pc.inputRecycleChan), "count") message.NewIntField(msg, "InChanLength", len(pc.inputRecycleChan), "count") msg.SetType("heka.input-report") message.NewStringField(msg, "name", "inputRecycleChan") message.NewStringField(msg, "key", "globals") reportChan <- pack pack = <-pc.reportRecycleChan msg = pack.Message message.NewIntField(msg, "InChanCapacity", cap(pc.injectRecycleChan), "count") message.NewIntField(msg, "InChanLength", len(pc.injectRecycleChan), "count") msg.SetType("heka.inject-report") message.NewStringField(msg, "name", "injectRecycleChan") message.NewStringField(msg, "key", "globals") reportChan <- pack pack = <-pc.reportRecycleChan msg = pack.Message message.NewIntField(msg, "InChanCapacity", cap(pc.router.InChan()), "count") message.NewIntField(msg, "InChanLength", len(pc.router.InChan()), "count") message.NewInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&pc.router.processMessageCount), "count") msg.SetType("heka.router-report") message.NewStringField(msg, "name", "Router") message.NewStringField(msg, "key", "globals") reportChan <- pack getReport := func(runner PluginRunner) (pack *PipelinePack) { pack = <-pc.reportRecycleChan if err = PopulateReportMsg(runner, pack.Message); err != nil { msg = pack.Message f, e = message.NewField("Error", err.Error(), "") if e == nil { msg.AddField(f) } msg.SetType("heka.plugin-report") } return } pc.inputsLock.Lock() for name, runner := range pc.InputRunners { if runner.Transient() { continue } pack = getReport(runner) message.NewStringField(pack.Message, "name", name) message.NewStringField(pack.Message, "key", "inputs") reportChan <- pack } pc.inputsLock.Unlock() for _, runner := range pc.allDecoders { pack = getReport(runner) message.NewStringField(pack.Message, "name", runner.Name()) message.NewStringField(pack.Message, "key", "decoders") reportChan <- pack } pc.filtersLock.Lock() for name, runner := range pc.FilterRunners { pack = getReport(runner) message.NewStringField(pack.Message, "name", name) message.NewStringField(pack.Message, "key", "filters") reportChan <- pack } pc.filtersLock.Unlock() for name, runner := range pc.OutputRunners { pack = getReport(runner) message.NewStringField(pack.Message, "name", name) message.NewStringField(pack.Message, "key", "outputs") reportChan <- pack } close(reportChan) }
func add_field(msg *message.Message, field_name string, value interface{}) { f, _ := message.NewField(field_name, value, message.Field_RAW) msg.AddField(f) }