コード例 #1
0
ファイル: encoders_test.go プロジェクト: RogerBai/heka
func getTestMessageWithFunnyFields() *message.Message {
	field, _ := message.NewField(`"foo`, "bar\n", "")
	field1, _ := message.NewField(`"number`, 64, "")
	field2, _ := message.NewField("\xa3", "\xa3", "")
	field3, _ := message.NewField("idField", "1234", "")

	msg := &message.Message{}
	msg.SetType("TEST")
	loc, _ := time.LoadLocation("UTC")
	t, _ := time.ParseInLocation("2006-01-02T15:04:05.000Z", "2013-07-16T15:49:05.070Z",
		loc)
	msg.SetTimestamp(t.UnixNano())
	msg.SetUuid(uuid.Parse("87cf1ac2-e810-4ddf-a02d-a5ce44d13a85"))
	msg.SetLogger("GoSpec")
	msg.SetSeverity(int32(6))
	msg.SetPayload("Test Payload")
	msg.SetEnvVersion("0.8")
	msg.SetPid(14098)
	msg.SetHostname("hostname")
	msg.AddField(field)
	msg.AddField(field1)
	msg.AddField(field2)
	msg.AddField(field3)

	return msg
}
コード例 #2
0
ファイル: sandbox_decoder.go プロジェクト: KushalP/heka
func (s *SandboxDecoder) SetDecoderRunner(dr DecoderRunner) {
	s.sb.InjectMessage(func(payload, payload_type, payload_name string) int {
		if len(payload_type) == 0 { // heka protobuf message
			t := s.pack.Message.GetType()
			h := s.pack.Message.GetHostname()
			l := s.pack.Message.GetLogger()
			if nil != proto.Unmarshal([]byte(payload), s.pack.Message) {
				return 1
			}
			if s.pack.Message.GetType() == "" {
				s.pack.Message.SetType(t)
			}
			if s.pack.Message.GetHostname() == "" {
				s.pack.Message.SetHostname(h)
			}
			if s.pack.Message.GetLogger() == "" {
				s.pack.Message.SetLogger(l)
			}
		} else {
			s.pack.Message.SetPayload(payload)
			ptype, _ := message.NewField("payload_type", payload_type, "file-extension")
			s.pack.Message.AddField(ptype)
			pname, _ := message.NewField("payload_name", payload_name, "")
			s.pack.Message.AddField(pname)
		}
		return 0
	})
}
コード例 #3
0
ファイル: file_polling_input.go プロジェクト: salekseev/heka
func (input *FilePollingInput) Run(runner pipeline.InputRunner,
	helper pipeline.PluginHelper) error {

	var (
		data    []byte
		pack    *pipeline.PipelinePack
		dRunner pipeline.DecoderRunner
		ok      bool
		err     error
	)

	if input.DecoderName != "" {
		if dRunner, ok = helper.DecoderRunner(input.DecoderName,
			fmt.Sprintf("%s-%s", runner.Name(), input.DecoderName)); !ok {
			return fmt.Errorf("Decoder not found: %s", input.DecoderName)
		}
		input.decoderChan = dRunner.InChan()
	}
	input.runner = runner

	hostname := helper.PipelineConfig().Hostname()
	packSupply := runner.InChan()
	tickChan := runner.Ticker()

	for {
		select {
		case <-input.stop:
			return nil
		case <-tickChan:
		}

		data, err = ioutil.ReadFile(input.FilePath)
		if err != nil {
			runner.LogError(fmt.Errorf("Error reading file: %s", err))
			continue
		}

		pack = <-packSupply
		pack.Message.SetUuid(uuid.NewRandom())
		pack.Message.SetTimestamp(time.Now().UnixNano())
		pack.Message.SetType("heka.file.polling")
		pack.Message.SetHostname(hostname)
		pack.Message.SetPayload(string(data))
		if field, err := message.NewField("TickerInterval", int(input.TickerInterval), ""); err != nil {
			runner.LogError(err)
		} else {
			pack.Message.AddField(field)
		}
		if field, err := message.NewField("FilePath", input.FilePath, ""); err != nil {
			runner.LogError(err)
		} else {
			pack.Message.AddField(field)
		}
		input.sendPack(pack)
	}

	return nil
}
コード例 #4
0
ファイル: main.go プロジェクト: johntdyer/golang-devops-stuff
func main() {
	configFile := flag.String("config", "sbmgr.toml", "Sandbox manager configuration file")
	scriptFile := flag.String("script", "xyz.lua", "Sandbox script file")
	scriptConfig := flag.String("scriptconfig", "xyz.toml", "Sandbox script configuration file")
	filterName := flag.String("filtername", "filter", "Sandbox filter name (used on unload)")
	action := flag.String("action", "load", "Sandbox manager action")
	flag.Parse()

	var config SbmgrConfig
	if _, err := toml.DecodeFile(*configFile, &config); err != nil {
		log.Printf("Error decoding config file: %s", err)
		return
	}
	sender, err := client.NewNetworkSender("tcp", config.IpAddress)
	if err != nil {
		log.Fatalf("Error creating sender: %s\n", err.Error())
	}
	encoder := client.NewProtobufEncoder(&config.Signer)
	manager := client.NewClient(sender, encoder)

	hostname, _ := os.Hostname()
	msg := &message.Message{}
	msg.SetType("heka.control.sandbox")
	msg.SetTimestamp(time.Now().UnixNano())
	msg.SetUuid(uuid.NewRandom())
	msg.SetHostname(hostname)

	switch *action {
	case "load":
		code, err := ioutil.ReadFile(*scriptFile)
		if err != nil {
			log.Printf("Error reading scriptFile: %s\n", err.Error())
			return
		}
		msg.SetPayload(string(code))
		conf, err := ioutil.ReadFile(*scriptConfig)
		if err != nil {
			log.Printf("Error reading scriptConfig: %s\n", err.Error())
			return
		}
		f, _ := message.NewField("config", string(conf), "toml")
		msg.AddField(f)
	case "unload":
		f, _ := message.NewField("name", *filterName, "")
		msg.AddField(f)
	default:
		log.Printf("Invalid action: %s", *action)
	}

	f1, _ := message.NewField("action", *action, "")
	msg.AddField(f1)
	err = manager.SendMessage(msg)
	if err != nil {
		log.Printf("Error sending message: %s\n", err.Error())
	}
}
コード例 #5
0
func getSentryPack() (pack *pipeline.PipelinePack) {
	recycleChan := make(chan *pipeline.PipelinePack, 1)
	pack = pipeline.NewPipelinePack(recycleChan)
	pack.Message.SetType("sentry")
	fTimeStamp, _ := message.NewField("epoch_timestamp", EPOCH_TS, "utc-seconds")
	fDsn, _ := message.NewField("dsn", DSN, "uri")
	pack.Message.AddField(fTimeStamp)
	pack.Message.AddField(fDsn)
	pack.Message.SetPayload(PAYLOAD)
	pack.Decoded = true
	return
}
コード例 #6
0
ファイル: http_listen_input.go プロジェクト: RogerBai/heka
func (hli *HttpListenInput) RequestHandler(w http.ResponseWriter, req *http.Request) {

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		fmt.Errorf("[HttpListenInput] Read HTTP request body fail: %s\n", err.Error())
	}
	req.Body.Close()

	unEscapedBody, _ := url.QueryUnescape(string(body))

	pack := <-hli.ir.InChan()
	pack.Message.SetUuid(uuid.NewRandom())
	pack.Message.SetTimestamp(time.Now().UnixNano())
	pack.Message.SetType("heka.httpdata.request")
	pack.Message.SetLogger(hli.ir.Name())
	pack.Message.SetHostname(req.RemoteAddr)
	pack.Message.SetPid(int32(os.Getpid()))
	pack.Message.SetSeverity(int32(6))
	pack.Message.SetPayload(unEscapedBody)
	if field, err := message.NewField("Protocol", req.Proto, ""); err == nil {
		pack.Message.AddField(field)
	} else {
		hli.ir.LogError(fmt.Errorf("can't add field: %s", err))
	}
	if field, err := message.NewField("UserAgent", req.UserAgent(), ""); err == nil {
		pack.Message.AddField(field)
	} else {
		hli.ir.LogError(fmt.Errorf("can't add field: %s", err))
	}
	if field, err := message.NewField("ContentType", req.Header.Get("Content-Type"), ""); err == nil {
		pack.Message.AddField(field)
	} else {
		hli.ir.LogError(fmt.Errorf("can't add field: %s", err))
	}

	for key, values := range req.URL.Query() {
		for i := range values {
			value := values[i]
			if field, err := message.NewField(key, value, ""); err == nil {
				pack.Message.AddField(field)
			} else {
				hli.ir.LogError(fmt.Errorf("can't add field: %s", err))
			}
		}
	}

	if hli.dRunner == nil {
		hli.ir.Inject(pack)
	} else {
		hli.dRunner.InChan() <- pack
	}
}
コード例 #7
0
func getStatsdPack(typeStr string, payload string) (pack *pipeline.PipelinePack) {
	recycleChan := make(chan *pipeline.PipelinePack, 1)
	pack = pipeline.NewPipelinePack(recycleChan)
	pack.Message.SetType(typeStr)
	pack.Message.SetLogger("thenamespace")
	fName, _ := message.NewField("name", "myname", "")
	fRate, _ := message.NewField("rate", .30, "")
	pack.Message.AddField(fName)
	pack.Message.AddField(fRate)
	pack.Message.SetPayload(payload)
	pack.Decoded = true
	return pack
}
コード例 #8
0
ファイル: geoip_decoder_test.go プロジェクト: orangemi/heka
func GeoIpDecoderSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := NewPipelineConfig(nil)
	pConfig.Globals.ShareDir = "/foo/bar/baz"

	c.Specify("A GeoIpDecoder", func() {
		decoder := new(GeoIpDecoder)
		decoder.SetPipelineConfig(pConfig)
		rec := new(geoip.GeoIPRecord)
		conf := decoder.ConfigStruct().(*GeoIpDecoderConfig)

		c.Expect(conf.DatabaseFile, gs.Equals, "/foo/bar/baz/GeoLiteCity.dat")

		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		nf, _ := message.NewField("remote_host", "74.125.142.147", "")
		pack.Message.AddField(nf)

		decoder.SourceIpField = "remote_host"
		conf.SourceIpField = "remote_host"
		decoder.Init(conf)

		rec.CountryCode = "US"
		rec.CountryCode3 = "USA"
		rec.CountryName = "United States"
		rec.Region = "CA"
		rec.City = "Mountain View"
		rec.PostalCode = "94043"
		rec.Latitude = 37.4192
		rec.Longitude = -122.0574
		rec.AreaCode = 650
		rec.CharSet = 1
		rec.ContinentCode = "NA"

		c.Specify("Test GeoIpDecoder Output", func() {
			buf := decoder.GeoBuff(rec)
			nf, _ = message.NewField("geoip", buf.Bytes(), "")
			pack.Message.AddField(nf)

			b, ok := pack.Message.GetFieldValue("geoip")
			c.Expect(ok, gs.IsTrue)

			c.Expect(string(b.([]byte)), gs.Equals, `{"latitude":37.4192008972168,"longitude":-122.0574035644531,"location":[-122.0574035644531,37.4192008972168],"coordinates":["-122.0574035644531","37.4192008972168"],"countrycode":"US","countrycode3":"USA","countryname":"United States","region":"CA","city":"Mountain View","postalcode":"94043","areacode":650,"charset":1,"continentcode":"NA"}`)
		})

	})
}
コード例 #9
0
ファイル: lua_sandbox_test.go プロジェクト: Nitro/heka
func getTestMessage() *message.Message {
	hostname, _ := os.Hostname()
	field, _ := message.NewField("foo", "bar", "")
	msg := &message.Message{}
	msg.SetType("TEST")
	msg.SetTimestamp(5123456789)
	msg.SetPid(9283)
	msg.SetUuid(uuid.NewRandom())
	msg.SetLogger("GoSpec")
	msg.SetSeverity(int32(6))
	msg.SetEnvVersion("0.8")
	msg.SetPid(int32(os.Getpid()))
	msg.SetHostname(hostname)
	msg.AddField(field)

	var emptyByte []byte
	data := []byte("data")
	field1, _ := message.NewField("bytes", data, "")
	field2, _ := message.NewField("int", int64(999), "")
	field2.AddValue(int64(1024))
	field3, _ := message.NewField("double", float64(99.9), "")
	field4, _ := message.NewField("bool", true, "")
	field5, _ := message.NewField("foo", "alternate", "")
	field6, _ := message.NewField("false", false, "")
	field7, _ := message.NewField("empty_bytes", emptyByte, "")
	msg.AddField(field1)
	msg.AddField(field2)
	msg.AddField(field3)
	msg.AddField(field4)
	msg.AddField(field5)
	msg.AddField(field6)
	msg.AddField(field7)
	return msg
}
コード例 #10
0
func SentryOutputSpec(c gs.Context) {

	pack := getSentryPack()
	output := new(SentryOutput)
	output.Init(output.ConfigStruct())
	sentryMsg := &SentryMsg{
		dataPacket: make([]byte, 0, output.config.MaxSentryBytes),
	}
	var err error

	c.Specify("verify data_packet bytes", func() {
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err, gs.Equals, nil)

		actual := string(sentryMsg.dataPacket)
		ts := int64(EPOCH_TS * 1e9)
		t := time.Unix(ts/1e9, ts%1e9)
		expected := fmt.Sprintf("Sentry sentry_timestamp=%s, "+
			"sentry_client=raven-go/1.0, sentry_version=2.0, "+
			"sentry_key=username\n\n%s", t.Format(time.RFC3339Nano), PAYLOAD)

		c.Expect(actual, gs.Equals, expected)
	})

	c.Specify("missing or invalid epoch_timestamp doesn't kill process", func() {
		f := pack.Message.FindFirstField("epoch_timestamp")
		*f.Name = "other"
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err.Error(), gs.Equals, "no `epoch_timestamp` field")

		f, _ = message.NewField("epoch_timestamp", "foo", "")
		pack.Message.AddField(f)
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err.Error(), gs.Equals, "`epoch_timestamp` isn't a float64")
	})

	c.Specify("missing or invalid dsn doesn't kill process", func() {
		f := pack.Message.FindFirstField("dsn")
		*f.Name = "other"
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err.Error(), gs.Equals, "no `dsn` field")

		f, _ = message.NewField("dsn", 42, "")
		pack.Message.AddField(f)
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err.Error(), gs.Equals, "`dsn` isn't a string")
	})

}
コード例 #11
0
func (f *StreamAggregatorFilter) committer(fr FilterRunner, h PluginHelper, wg *sync.WaitGroup) {
	initBatch := make([]byte, 0, 10000)
	f.backChan <- initBatch
	var (
		tag      string
		outBatch []byte
	)
	tag = f.StreamAggregatorTag

	for outBatch = range f.batchChan {
		pack, e := h.PipelinePack(f.msgLoopCount)
		if e != nil {
			fr.LogError(e)
			break
		}

		tagField, _ := message.NewField("StreamAggregatorTag", tag, "")
		pack.Message.AddField(tagField)
		pack.Message.SetUuid(uuid.NewRandom())
		if f.OutputType == "json" {
			jsonStr := strings.TrimRight(string(outBatch), ",")
			pack.Message.SetPayload("[" + jsonStr + "]")
		} else {
			pack.Message.SetPayload(string(outBatch))
		}
		fr.Inject(pack)

		outBatch = outBatch[:0]
		f.backChan <- outBatch
	}
	wg.Done()
}
コード例 #12
0
ファイル: zlib_filter.go プロジェクト: huhongbo/heka-zlib
func (f *ZlibFilter) committer(fr FilterRunner, h PluginHelper, wg *sync.WaitGroup) {
	initBatch := make([]byte, 0, 10000)
	f.backChan <- initBatch
	var (
		tag string
		//ok bool
		outBatch []byte
	)
	tag = f.ZlibTag

	for outBatch = range f.batchChan {
		pack, e := h.PipelinePack(f.msgLoopCount)
		if e != nil {
			fr.LogError(e)
			break
		}
		var b bytes.Buffer
		w := zlib.NewWriter(&b)
		w.Write(outBatch)
		w.Close()

		tagField, _ := message.NewField("ZlibTag", tag, "")
		pack.Message.AddField(tagField)
		pack.Message.SetUuid(uuid.NewRandom())
		pack.Message.SetPayload(b.String())
		fr.Inject(pack)

		outBatch = outBatch[:0]
		f.backChan <- outBatch
	}
	wg.Done()
}
コード例 #13
0
ファイル: geoip_decoder.go プロジェクト: orangemi/heka
func (ld *GeoIpDecoder) Decode(pack *PipelinePack) (packs []*PipelinePack, err error) {
	var buf bytes.Buffer
	var ipAddr, _ = pack.Message.GetFieldValue(ld.SourceIpField)

	ip, ok := ipAddr.(string)

	if !ok {
		// IP field was not a string. Field could just be blank. Return without error.
		packs = []*PipelinePack{pack}
		return
	}

	if ld.gi != nil {
		rec := ld.gi.GetRecord(ip)
		if rec != nil {
			buf = ld.GeoBuff(rec)
		} else {
			// IP address did not return a valid GeoIp record but that's ok sometimes(private ip?). Return without error.
			packs = []*PipelinePack{pack}
			return
		}
	}

	if buf.Len() > 0 {
		var nf *message.Field
		nf, err = message.NewField(ld.TargetField, buf.Bytes(), "")
		pack.Message.AddField(nf)
	}

	packs = []*PipelinePack{pack}

	return
}
コード例 #14
0
ファイル: report.go プロジェクト: KushalP/heka
// Convenience function for creating and setting a string field called "name"
// on a message object.
func newStringField(msg *message.Message, name string, val string) {
	if f, err := message.NewField(name, val, ""); err == nil {
		msg.AddField(f)
	} else {
		fmt.Println("Report error adding string field: ", err)
	}
}
コード例 #15
0
ファイル: decoders.go プロジェクト: pchojnacki/heka
// Applies this message template's values to the provided message object,
// interpolating the provided substitutions into the values in the process.
func (mt MessageTemplate) PopulateMessage(msg *message.Message, subs map[string]string) error {
	var val string
	for field, rawVal := range mt {
		val = InterpolateString(rawVal, subs)
		switch field {
		case "Logger":
			msg.SetLogger(val)
		case "Type":
			msg.SetType(val)
		case "Payload":
			msg.SetPayload(val)
		case "Hostname":
			msg.SetHostname(val)
		case "Pid":
			pid, err := strconv.ParseInt(val, 10, 32)
			if err != nil {
				return err
			}
			msg.SetPid(int32(pid))
		case "Uuid":
			msg.SetUuid([]byte(val))
		default:
			fi := strings.SplitN(field, "|", 2)
			if len(fi) < 2 {
				fi = append(fi, "")
			}
			f, err := message.NewField(fi[0], val, fi[1])
			msg.AddField(f)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
コード例 #16
0
ファイル: report.go プロジェクト: KushalP/heka
// Convenience function for creating a new int64 field on a message object.
func newInt64Field(msg *message.Message, name string, val int64, representation string) {
	if f, err := message.NewField(name, val, representation); err == nil {
		msg.AddField(f)
	} else {
		fmt.Println("Report error adding int64 field: ", err)
	}
}
コード例 #17
0
ファイル: http_listen_input.go プロジェクト: Nitro/heka
func (hli *HttpListenInput) makeField(name string, value string) (field *message.Field, err error) {
	field, err = message.NewField(name, value, "")
	if err != nil {
		hli.ir.LogError(fmt.Errorf("can't add field %s: %s", name, err))
	}
	return
}
コード例 #18
0
ファイル: report.go プロジェクト: hellcoderz/heka
func PopulateReportMsg(pr PluginRunner, msg *message.Message) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("'%s' `populateReportMsg` panic: %s", pr.Name(), r)
		}
	}()

	if reporter, ok := pr.Plugin().(ReportingPlugin); ok {
		if err = reporter.ReportMsg(msg); err != nil {
			return
		}
	}

	if fRunner, ok := pr.(FilterRunner); ok {
		newIntField(msg, "InChanCapacity", cap(fRunner.InChan()))
		newIntField(msg, "InChanLength", len(fRunner.InChan()))
	} else if dRunner, ok := pr.(DecoderRunner); ok {
		newIntField(msg, "InChanCapacity", cap(dRunner.InChan()))
		newIntField(msg, "InChanLength", len(dRunner.InChan()))
	}

	if msg.GetType() != "" {
		var f *message.Field
		f, err = message.NewField("Type", msg.GetType(), message.Field_RAW)
		if err != nil {
			return
		}
		msg.AddField(f)
	}
	msg.SetType("heka.plugin-report")
	return
}
コード例 #19
0
ファイル: process_input.go プロジェクト: Nitro/heka
func (pi *ProcessInput) initDelivery(streamName string) (Deliverer, SplitterRunner) {
	deliverer := pi.ir.NewDeliverer(streamName)
	sRunner := pi.ir.NewSplitterRunner(streamName)
	if !sRunner.UseMsgBytes() {
		packDecorator := func(pack *PipelinePack) {
			pack.Message.SetType("ProcessInput")
			pack.Message.SetPid(pi.hekaPid)
			pack.Message.SetHostname(pi.hostname)
			// Add ProcessInputName
			fPInputName, err := message.NewField("ProcessInputName",
				fmt.Sprintf("%s.%s", pi.ProcessName, streamName), "")
			if err == nil {
				pack.Message.AddField(fPInputName)
			} else {
				pi.ir.LogError(err)
			}
			// Wait for the result for subcommands.
			// Add exit status and subcommand error messages to pack.
			var r int
			if exiterr, ok := pi.ccStatus.ExitStatus.(*exec.ExitError); ok {
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					r = status.ExitStatus()
				}
			}

			exitStatus, err := message.NewField("ExitStatus", r, "")
			if err == nil {
				pack.Message.AddField(exitStatus)
			} else {
				pi.ir.LogError(err)
			}

			if pi.ccStatus.SubcmdErrors != nil {
				subcmdStatus, err := message.NewField("SubcmdErrors", pi.ccStatus.SubcmdErrors.Error(), "")
				if err == nil {
					pack.Message.AddField(subcmdStatus)
				} else {
					pi.ir.LogError(err)
				}
			}
		}
		sRunner.SetPackDecorator(packDecorator)
	}

	return deliverer, sRunner
}
コード例 #20
0
ファイル: sqs3_input.go プロジェクト: relud/heka_plugin_sqs3
func (input *Sqs3Input) packDecorator(pack *pipeline.PipelinePack) {
    field, err := message.NewField("SqsQueue", int(input.SqsQueue), "")
    if err != nil {
        input.runner.LogError(
            fmt.Errorf("can't add 'SqsQueue' field: %s", err.Error()))
    } else {
        pack.Message.AddField(field)
    }
}
コード例 #21
0
ファイル: kafka_input.go プロジェクト: intoximeters/heka
func (k *KafkaInput) addField(pack *pipeline.PipelinePack, name string,
	value interface{}, representation string) {

	if field, err := message.NewField(name, value, representation); err == nil {
		pack.Message.AddField(field)
	} else {
		k.ir.LogError(fmt.Errorf("can't add '%s' field: %s", name, err.Error()))
	}
}
コード例 #22
0
ファイル: plugin_runners.go プロジェクト: Nitro/heka
// AddDecodeFailureFields adds two fields to the provided message object. The
// first field is a boolean field called `decode_failure`, set to true. The
// second is a string field called `decode_error` which will contain the
// provided error message, truncated to 500 bytes if necessary.
func AddDecodeFailureFields(m *message.Message, errMsg string) error {
	field0, err := message.NewField("decode_failure", true, "")
	if err != nil {
		err = fmt.Errorf("field creation error: %s", err.Error())
		return err
	}
	if len(errMsg) > 500 {
		errMsg = errMsg[:500]
	}
	field1, err := message.NewField("decode_error", errMsg, "")
	if err != nil {
		err = fmt.Errorf("field creation error: %s", err.Error())
		return err
	}
	m.AddField(field0)
	m.AddField(field1)
	return nil
}
コード例 #23
0
func newField(pack *pipeline.PipelinePack, name string, value interface{}) {
	var field *message.Field
	var err error
	if field, err = message.NewField(name, value, ""); err == nil {
		pack.Message.AddField(field)
	} else {
		log.Println("CloudwatchInput can't add field: ", name)
	}
}
コード例 #24
0
ファイル: file_polling_input.go プロジェクト: orangemi/heka
func (input *FilePollingInput) packDecorator(pack *pipeline.PipelinePack) {
	pack.Message.SetType("heka.file.polling")
	pack.Message.SetHostname(input.hostname)

	field, err := message.NewField("TickerInterval", int(input.TickerInterval), "")
	if err != nil {
		input.runner.LogError(
			fmt.Errorf("can't add 'TickerInterval' field: %s", err.Error()))
	} else {
		pack.Message.AddField(field)
	}
	field, err = message.NewField("FilePath", input.FilePath, "")
	if err != nil {
		input.runner.LogError(
			fmt.Errorf("can't add 'FilePath' field: %s", err.Error()))
	} else {
		pack.Message.AddField(field)
	}
}
コード例 #25
0
func (d *StatsToFieldsDecoder) addStatField(pack *PipelinePack, name string,
	value interface{}) error {

	field, err := message.NewField(name, value, "")
	if err != nil {
		return fmt.Errorf("error adding field '%s': %s", name, err)
	}
	pack.Message.AddField(field)
	return nil
}
コード例 #26
0
ファイル: message_template.go プロジェクト: orangemi/heka
// Applies this message template's values to the provided message object,
// interpolating the provided substitutions into the values in the process.
func (mt MessageTemplate) PopulateMessage(msg *message.Message, subs map[string]string) error {
	var val string
	for field, rawVal := range mt {
		if subs == nil {
			val = rawVal
		} else {
			val = InterpolateString(rawVal, subs)
		}
		switch field {
		case "Logger":
			msg.SetLogger(val)
		case "Type":
			msg.SetType(val)
		case "Payload":
			msg.SetPayload(val)
		case "Hostname":
			msg.SetHostname(val)
		case "Pid":
			intPart := strings.Split(val, ".")[0]
			pid, err := strconv.ParseInt(intPart, 10, 32)
			if err != nil {
				return err
			}
			msg.SetPid(int32(pid))
		case "Severity":
			severity, err := strconv.ParseInt(val, 10, 32)
			if err != nil {
				return err
			}
			msg.SetSeverity(int32(severity))
		case "Uuid":
			if len(val) == message.UUID_SIZE {
				msg.SetUuid([]byte(val))
			} else {
				if uuidBytes := uuid.Parse(val); uuidBytes == nil {
					return errors.New("Invalid UUID string.")
				} else {
					msg.SetUuid(uuidBytes)
				}
			}
		default:
			fi := strings.SplitN(field, "|", 2)
			if len(fi) < 2 {
				fi = append(fi, "")
			}
			f, err := message.NewField(fi[0], val, fi[1])
			msg.AddField(f)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
コード例 #27
0
ファイル: heka_log.go プロジェクト: nikhilm/pushgo
//TODO: Change the last arg to be something like fields ...interface{}
func (self HekaLogger) Log(level int32, mtype, payload string, fields JsMap) (err error) {

	if len(fields) > 0 {
		log.Printf("[%d]% 7s: %s %s", level, mtype, payload, fields)
	} else {
		log.Printf("[%d]% 7s: %s", level, mtype, payload)
	}

	// Don't send an error if there's nothing to do
	if self.sender == nil {
		return nil
	}

	var stream []byte

	msg := &message.Message{}
	msg.SetTimestamp(time.Now().UnixNano())
	msg.SetUuid(uuid.NewRandom())
	msg.SetLogger(self.logname)
	msg.SetType(mtype)
	msg.SetPid(self.pid)
	msg.SetSeverity(level)
	msg.SetHostname(self.hostname)
	if len(payload) > 0 {
		msg.SetPayload(payload)
	}
	for key, ival := range fields {
		if ival == nil {
			continue
		}
		if key == "" {
			continue
		}
		field, err := message.NewField(key, ival, message.Field_RAW)
		if err != nil {
			log.Fatal("ERROR: Could not log field %s:%s (%s)", field,
				ival.(string), err)
			return err
		}
		msg.AddField(field)
	}
	err = self.encoder.EncodeMessageStream(msg, &stream)
	if err != nil {
		log.Fatal("ERROR: Could not encode log message (%s)", err)
		return err
	}
	err = self.sender.SendMessage(stream)
	if err != nil {
		log.Fatal("ERROR: Could not send message (%s)", err)
		return err
	}
	return nil
}
コード例 #28
0
func TestGetMessageVariable(t *testing.T) {
	field, _ := message.NewField("foo", "bar", "")
	msg := &message.Message{}
	msg.SetType("TEST")
	msg.SetLogger("GoSpec")
	msg.SetHostname("example.com")
	msg.SetPayload("xxx yyy")
	msg.AddField(field)
	data := []byte("data")
	field1, _ := message.NewField("bytes", data, "")
	field2, _ := message.NewField("int", int64(999), "")
	field2.AddValue(int64(1024))
	field3, _ := message.NewField("double", float64(99.9), "")
	field4, _ := message.NewField("bool", true, "")
	field5, _ := message.NewField("foo", "alternate", "")
	msg.AddField(field1)
	msg.AddField(field2)
	msg.AddField(field3)
	msg.AddField(field4)
	msg.AddField(field5)

	tests := []string{
		"Type",
		"Logger",
		"Hostname",
		"Payload",
		"Fields[foo]",
		"Fields[bytes]",
		"Fields[int]",
		"Fields[double]",
		"Fields[bool]",
		"Fields[foo][1]",
		"Fields[int][0][1]",
	}
	results := []string{
		"TEST",
		"GoSpec",
		"example.com",
		"xxx yyy",
		"bar",
		"data",
		"999",
		"99.9",
		"true",
		"alternate",
		"1024",
	}

	for i, v := range tests {
		mvar := verifyMessageVariable(v)
		if mvar == nil {
			t.Errorf("verification failed %s", v)
			return
		}
		s := getMessageVariable(msg, mvar)
		if s != results[i] {
			t.Errorf("%s Expected: %s Received: %s", v, results[i], s)
		}
	}
}
コード例 #29
0
ファイル: lua_sandbox_test.go プロジェクト: hellcoderz/heka
func getTestMessage() *message.Message {
	hostname, _ := os.Hostname()
	field, _ := message.NewField("foo", "bar", message.Field_RAW)
	msg := &message.Message{}
	msg.SetType("TEST")
	msg.SetTimestamp(time.Now().UnixNano())
	msg.SetUuid(uuid.NewRandom())
	msg.SetLogger("GoSpec")
	msg.SetSeverity(int32(6))
	msg.SetEnvVersion("0.8")
	msg.SetPid(int32(os.Getpid()))
	msg.SetHostname(hostname)
	msg.AddField(field)

	data := []byte("data")
	field1, _ := message.NewField("bytes", data, message.Field_RAW)
	field2, _ := message.NewField("int", int64(999), message.Field_RAW)
	field2.AddValue(int64(1024))
	field3, _ := message.NewField("double", float64(99.9), message.Field_RAW)
	field4, _ := message.NewField("bool", true, message.Field_RAW)
	field5, _ := message.NewField("foo", "alternate", message.Field_RAW)
	msg.AddField(field1)
	msg.AddField(field2)
	msg.AddField(field3)
	msg.AddField(field4)
	msg.AddField(field5)
	return msg
}
コード例 #30
0
ファイル: heka.go プロジェクト: joedivita/procd
func (self HekaClient) SendCPUResources(pl *resources.Collection, ipl bool) (err error) {
	var stream []byte

	msg := &message.Message{}
	msg.SetTimestamp(pl.Timestamp.UnixNano())
	msg.SetUuid(uuid.NewRandom())
	msg.SetType("procd.resources.cpu")
	msg.SetLogger("procd")
	msg.SetPid(self.pid)
	msg.SetSeverity(int32(6))
	msg.SetHostname(self.hostname)

	if ipl == true {
		marshalledPayload, _ := json.MarshalIndent(pl, "", " ")
		msg.SetPayload(string(marshalledPayload))
	}

	// Super awful way to do this, should use reflection but need to test quickly
	f, _ := message.NewField("CpuUser", pl.CPU.User, "")
	msg.AddField(f)
	f, _ = message.NewField("CpuNice", pl.CPU.Nice, "")
	msg.AddField(f)
	f, _ = message.NewField("CpuIdle", pl.CPU.Idle, "")
	msg.AddField(f)
	f, _ = message.NewField("CpuIowait", pl.CPU.IOWait, "")
	msg.AddField(f)
	f, _ = message.NewField("CpuIrq", pl.CPU.IRQ, "")
	msg.AddField(f)
	f, _ = message.NewField("CpuSoftirq", pl.CPU.SoftIRQ, "")
	msg.AddField(f)
	f, _ = message.NewField("CpuSteal", pl.CPU.Steal, "")
	msg.AddField(f)
	f, _ = message.NewField("CpuGuest", pl.CPU.Guest, "")
	msg.AddField(f)
	f, _ = message.NewField("CpuGuestnice", pl.CPU.GuestNice, "")
	msg.AddField(f)

	err = self.encoder.EncodeMessageStream(msg, &stream)
	verifyErrorResponse(err, "output[heka] encode message error")
	err = self.sender.SendMessage(stream)
	verifyErrorResponse(err, "output[heka] send message error")
	return nil
}