Exemplo n.º 1
0
func (rpsi *RedisPubSubInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error {
	var (
		dRunner pipeline.DecoderRunner
		decoder pipeline.Decoder
		pack    *pipeline.PipelinePack
		e       error
		ok      bool
	)
	// Get the InputRunner's chan to receive empty PipelinePacks
	packSupply := ir.InChan()

	if rpsi.conf.DecoderName != "" {
		if dRunner, ok = h.DecoderRunner(rpsi.conf.DecoderName, fmt.Sprintf("%s-%s", ir.Name(), rpsi.conf.DecoderName)); !ok {
			return fmt.Errorf("Decoder not found: %s", rpsi.conf.DecoderName)
		}
		decoder = dRunner.Decoder()
	}

	//Connect to the channel
	psc := redis.PubSubConn{Conn: rpsi.conn}
	psc.PSubscribe(rpsi.conf.Channel)

	for {
		switch n := psc.Receive().(type) {
		case redis.PMessage:
			// Grab an empty PipelinePack from the InputRunner
			pack = <-packSupply
			pack.Message.SetType("redis_pub_sub")
			pack.Message.SetLogger(n.Channel)
			pack.Message.SetPayload(string(n.Data))
			pack.Message.SetTimestamp(time.Now().UnixNano())
			var packs []*pipeline.PipelinePack
			if decoder == nil {
				packs = []*pipeline.PipelinePack{pack}
			} else {
				packs, e = decoder.Decode(pack)
			}
			if packs != nil {
				for _, p := range packs {
					ir.Inject(p)
				}
			} else {
				if e != nil {
					ir.LogError(fmt.Errorf("Couldn't parse Redis message: %s", n.Data))
				}
				pack.Recycle(nil)
			}
		case redis.Subscription:
			ir.LogMessage(fmt.Sprintf("Subscription: %s %s %d\n", n.Kind, n.Channel, n.Count))
			if n.Count == 0 {
				return errors.New("No channel to subscribe")
			}
		case error:
			fmt.Printf("error: %v\n", n)
			return n
		}
	}

	return nil
}
Exemplo n.º 2
0
func (f *UptimeFilter) Run(runner pipeline.FilterRunner, helper pipeline.PluginHelper) (err error) {
	var (
		pack    *pipeline.PipelinePack
		payload string
	)

	inChan := runner.InChan()
	for pack = range inChan {
		payload = pack.Message.GetPayload()
		runner.LogMessage("Payload: " + payload)
		if f.hours == nil {
			f.hours = make(map[int64]bool)
		}
		var epoch int64 = f.GetEpoch(payload)
		f.startHour, f.endHour = f.FigureOutStartAndEndHour(epoch)
		if !f.hours[f.startHour] {
			f.InitFilterForStartHour(f.startHour, payload)
		} else {
			f.CalculateUptimeFor(f.startHour, f.endHour)
			// f.hours[&f.startHour] = false
			log.Printf("Length of map: &d", len(f.hours))
		}
		log.Printf("Start hour: %d", f.startHour)
		log.Printf("End hour: %d", f.endHour)
		log.Printf("EPOCH: %d", epoch)
		pack.Recycle()
	}
	return
}
Exemplo n.º 3
0
func (rli *RedisListInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error {
	var (
		pack  *pipeline.PipelinePack
		packs []*pipeline.PipelinePack
	)

	// Get the InputRunner's chan to receive empty PipelinePacks
	inChan := ir.InChan()

	for {
		message, err := rli.conn.Do("RPOP", rli.conf.ListName)
		if err != nil {
			ir.LogError(fmt.Errorf("Redis RPOP error: %s", err))
			// TODO: should reconnect redis rather than close it
			rli.Stop()
			break
		}
		if message != nil {
			pack = <-inChan
			pack.Message.SetType("redis_list")
			pack.Message.SetPayload(string(message.([]uint8)))
			packs = []*pipeline.PipelinePack{pack}
			if packs != nil {
				for _, p := range packs {
					ir.Inject(p)
				}
			} else {
				pack.Recycle(nil)
			}
		} else {
			time.Sleep(time.Second)
		}
	}
	return nil
}
Exemplo n.º 4
0
func (rli *RedisInput) InsertMessage(ir pipeline.InputRunner, decoder pipeline.Decoder, msg string) {
	var (
		pack *pipeline.PipelinePack
		e    error
	)
	// Get the InputRunner's chan to receive empty PipelinePacks
	packSupply := ir.InChan()

	pack = <-packSupply
	pack.Message.SetType(rli.conf.Key)
	pack.Message.SetLogger("Redis")
	pack.Message.SetPayload(msg)
	pack.Message.SetTimestamp(time.Now().UnixNano())

	var packs []*pipeline.PipelinePack
	if decoder == nil {
		packs = []*pipeline.PipelinePack{pack}
	} else {
		packs, e = decoder.Decode(pack)
	}

	if packs != nil {
		for _, p := range packs {
			ir.Inject(p)
		}
	} else {
		if e != nil {
			ir.LogError(fmt.Errorf("Couldn't parse %s", msg))
			pack.Recycle(e)
		} else {
			pack.Recycle(nil)
			fmt.Println("pack recycle!")
		}
	}
}
Exemplo n.º 5
0
func (sop *SCAMPOutputPlugin) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	var pack *pipeline.PipelinePack

	// We have no default encoder
	if or.Encoder() == nil {
		return errors.New("Encoder required.")
	}

	for pack = range or.InChan() {
		scamp.Info.Printf("received pipeline pack")
		encoded, err := or.Encode(pack) // pack.Message.GetPayload()

		if err == nil {
			scamp.Info.Printf("payload: %s", encoded)
			msg := scamp.NewMessage()
			msg.SetEnvelope(scamp.ENVELOPE_JSON)
			msg.SetAction(sop.conf.Action)
			msg.SetVersion(1)
			msg.Write(encoded)
		}

		pack.Recycle(err)
	}
	fmt.Println("sup from end of for loop in Run")
	return
}
Exemplo n.º 6
0
func (output *IrcOutput) Run(runner pipeline.OutputRunner,
	helper pipeline.PluginHelper) error {
	if runner.Encoder() == nil {
		return errors.New("Encoder required.")
	}

	output.runner = runner

	// Register callbacks to handle events
	registerCallbacks(output)

	var err error

	// Connect to the Irc Server
	err = output.Conn.Connect(output.Server)
	if err != nil {
		return fmt.Errorf("Unable to connect to irc server %s: %s",
			output.Server, err)
	}

	// Start a goroutine for recieving messages, and throttling before sending
	// to the Irc Server
	output.wg.Add(1)
	go processOutQueue(output)

	var outgoing []byte
	ok := true
	inChan := runner.InChan()
	var pack *pipeline.PipelinePack
	for ok {
		select {
		case pack, ok = <-inChan:
		case <-output.die:
			ok = false
		}
		if !ok {
			break
		}
		outgoing, err = runner.Encode(pack)
		if err != nil {
			output.runner.LogError(err)
		} else if outgoing != nil {
			// Send the message to each irc channel. If the out queue is full,
			// then we need to drop the message and log an error.
			for i, ircChannel := range output.Channels {
				ircMsg := IrcMsg{outgoing, ircChannel, i}
				select {
				case output.OutQueue <- ircMsg:
				default:
					output.runner.LogError(ErrOutQueueFull)
				}
			}
		}
		pack.Recycle()
	}
	output.cleanup()
	return nil
}
Exemplo n.º 7
0
// Run runs the FileReadFilter filter, which inspects each message, and appends
// the content of the file named as the executed template to the existing payload.
// The resulting message will be injected back, and have newType type.
func (fr FileReadFilter) Run(r pipeline.FilterRunner, h pipeline.PluginHelper) (err error) {
	if fr.tmpl == nil {
		return errors.New("FileReadFilter: empty template")
	}
	var (
		fh           *os.File
		inp          io.Reader
		npack, opack *pipeline.PipelinePack
	)
	out := bytes.NewBuffer(make([]byte, 0, 4096))
	log.Printf("FileReadFilter: Starting with template %s", fr.tmpl)
	for opack = range r.InChan() {
		//log.Printf("opack=%v", opack)
		//if opack.Decoded {
		out.Reset()
		if err = fr.tmpl.Execute(out, extendedMessage{opack.Message}); err != nil {
			opack.Recycle()
			return fmt.Errorf("FileReadFilter: error executing template %v with message %v: %v",
				fr.tmpl, opack.Message, err)
		}
		//log.Printf("out=%q", out)
		if fh, err = os.Open(out.String()); err != nil {
			log.Printf("FileReadFilter: cannot read %q: %v", out, err)
			opack.Recycle()
			continue
		}
		out.Reset()
		//if _, err = io.Copy(out, io.LimitedReader{R: fh, N: 65000}); err != nil && err != io.EOF {
		inp = fh
		if fr.decoder != nil {
			inp = transform.NewReader(fh, fr.decoder)
		}
		if _, err = io.Copy(out, inp); err != nil && err != io.EOF {
			log.Printf("FileReadFilter: error reading %q: %v", fh.Name(), err)
			opack.Recycle()
			fh.Close()
			continue
		}
		fh.Close()

		npack = h.PipelinePack(opack.MsgLoopCount)
		if npack == nil {
			opack.Recycle()
			return errors.New("FileReadFilter: no output pack - infinite loop?")
		}
		npack.Decoded = true
		npack.Message = message.CopyMessage(opack.Message)
		npack.Message.SetType(fr.newType)
		npack.Message.SetPayload(npack.Message.GetPayload() + "\n" + out.String())
		if !r.Inject(npack) {
			log.Printf("FileReadFilter: cannot inject new pack %v", npack)
		}
		//}
		opack.Recycle()
	}
	return nil
}
func (clo *CloudLoggingOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	var (
		pack       *pipeline.PipelinePack
		e          error
		k          string
		m          *logging.LogEntry
		exist      bool
		ok         = true
		inChan     = or.InChan()
		groupBatch = make(map[string]*LogBatch)
		outBatch   *LogBatch
		ticker     = time.Tick(time.Duration(clo.conf.FlushInterval) * time.Millisecond)
	)
	clo.or = or
	go clo.committer()
	for ok {
		select {
		case pack, ok = <-inChan:
			// Closed inChan => we're shutting down, flush data.
			if !ok {
				clo.sendGroupBatch(groupBatch)
				close(clo.batchChan)
				<-clo.outputExit
				break
			}

			k, m, e = clo.Encode(pack)
			pack.Recycle()
			if e != nil {
				or.LogError(e)
				continue
			}

			if k != "" && m != nil {
				outBatch, exist = groupBatch[k]
				if !exist {
					outBatch = &LogBatch{count: 0, batch: make([]*logging.LogEntry, 0, 100), name: k}
					groupBatch[k] = outBatch
				}

				outBatch.batch = append(outBatch.batch, m)
				if outBatch.count++; clo.CheckFlush(int(outBatch.count), len(outBatch.batch)) {
					if len(outBatch.batch) > 0 {
						outBatch.batch = clo.sendBatch(k, outBatch.batch, outBatch.count)
						outBatch.count = 0
					}
				}
			}
		case <-ticker:
			clo.sendGroupBatch(groupBatch)
		case err = <-clo.outputExit:
			ok = false
		}
	}
	return
}
func (cmo *CloudMonitoringOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	var (
		pack     *pipeline.PipelinePack
		e        error
		m        *cloudmonitoring.TimeseriesPoint
		ok       = true
		count    int64
		inChan   = or.InChan()
		outBatch = make([]*cloudmonitoring.TimeseriesPoint, 0, 200)
		ticker   = time.Tick(time.Duration(cmo.conf.FlushInterval) * time.Millisecond)
	)
	cmo.or = or
	go cmo.committer()
	for ok {
		select {
		case pack, ok = <-inChan:
			// Closed inChan => we're shutting down, flush data.
			if !ok {
				if len(outBatch) > 0 {
					cmo.sendBatch(outBatch, count)
				}
				close(cmo.batchChan)
				<-cmo.outputExit
				break
			}

			m, e = cmo.Encode(pack)
			pack.Recycle()
			if e != nil {
				or.LogError(e)
				continue
			}

			if m != nil {
				outBatch = append(outBatch, m)

				if count++; cmo.CheckFlush(int(count), len(outBatch)) {
					if len(outBatch) > 0 {
						outBatch = cmo.sendBatch(outBatch, count)
						count = 0
					}
				}
			}
		case <-ticker:
			if len(outBatch) > 0 {
				outBatch = cmo.sendBatch(outBatch, count)
			}
			count = 0
		case err = <-cmo.outputExit:
			ok = false
		}
	}
	return
}
Exemplo n.º 10
0
func (cef *CefOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {

	var (
		facility, priority syslog.Priority
		ident              string
		ok                 bool
		p                  syslog.Priority
		e                  error
		pack               *pipeline.PipelinePack
	)
	syslogMsg := new(SyslogMsg)
	for pack = range or.InChan() {

		// default values
		facility, priority = syslog.LOG_LOCAL4, syslog.LOG_INFO
		ident = "heka_no_ident"

		priField := pack.Message.FindFirstField("cef_meta.syslog_priority")
		if priField != nil {
			priStr := priField.ValueString[0]
			if p, ok = SYSLOG_PRIORITY[priStr]; ok {
				priority = p
			}
		}

		facField := pack.Message.FindFirstField("cef_meta.syslog_facility")
		if facField != nil {
			facStr := facField.ValueString[0]
			if p, ok = SYSLOG_FACILITY[facStr]; ok {
				facility = p
			}
		}

		idField := pack.Message.FindFirstField("cef_meta.syslog_ident")
		if idField != nil {
			ident = idField.ValueString[0]
		}

		syslogMsg.priority = priority | facility
		syslogMsg.prefix = ident
		syslogMsg.payload = pack.Message.GetPayload()
		pack.Recycle()

		_, e = cef.syslogWriter.WriteString(syslogMsg.priority, syslogMsg.prefix,
			syslogMsg.payload)

		if e != nil {
			or.LogError(e)
		}
	}

	cef.syslogWriter.Close()
	return
}
Exemplo n.º 11
0
func (cwo *CloudwatchOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	inChan := or.InChan()

	payloads := make(chan CloudwatchDatapoints, cwo.backlog)
	go cwo.Submitter(payloads, or)

	var (
		pack          *pipeline.PipelinePack
		msg           *message.Message
		rawDataPoints *CloudwatchDatapointPayload
		dataPoints    *CloudwatchDatapoints
	)
	dataPoints = new(CloudwatchDatapoints)
	dataPoints.Datapoints = make([]cloudwatch.MetricDatum, 0, 0)

	for pack = range inChan {
		rawDataPoints = new(CloudwatchDatapointPayload)
		msg = pack.Message
		err = json.Unmarshal([]byte(msg.GetPayload()), rawDataPoints)
		if err != nil {
			or.LogMessage(fmt.Sprintf("warning, unable to parse payload: %s", err))
			err = nil
			continue
		}
		// Run through the list and convert them to CloudwatchDatapoints
		for _, rawDatum := range rawDataPoints.Datapoints {
			datum := cloudwatch.MetricDatum{
				Dimensions:      rawDatum.Dimensions,
				MetricName:      rawDatum.MetricName,
				Unit:            rawDatum.Unit,
				Value:           rawDatum.Value,
				StatisticValues: rawDatum.StatisticValues,
			}
			if rawDatum.Timestamp != "" {
				parsedTime, err := message.ForgivingTimeParse("", rawDatum.Timestamp, cwo.tzLocation)
				if err != nil {
					or.LogMessage(fmt.Sprintf("unable to parse timestamp for datum: %s", rawDatum))
					continue
				}
				datum.Timestamp = parsedTime
			}
			dataPoints.Datapoints = append(dataPoints.Datapoints, datum)
		}
		payloads <- *dataPoints
		dataPoints.Datapoints = dataPoints.Datapoints[:0]
		rawDataPoints.Datapoints = rawDataPoints.Datapoints[:0]
		pack.Recycle()
	}
	or.LogMessage("shutting down AWS Cloudwatch submitter")
	cwo.stopChan <- true
	<-cwo.stopChan
	return
}
Exemplo n.º 12
0
func (k *KafkaOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	defer func() {
		k.producer.Close()
		k.client.Close()
	}()

	if or.Encoder() == nil {
		return errors.New("Encoder required.")
	}

	inChan := or.InChan()
	errChan := k.producer.Errors()
	var wg sync.WaitGroup
	wg.Add(1)
	go k.processKafkaErrors(or, errChan, &wg)

	var (
		pack  *pipeline.PipelinePack
		topic = k.config.Topic
		key   sarama.Encoder
	)

	for pack = range inChan {
		atomic.AddInt64(&k.processMessageCount, 1)

		if k.topicVariable != nil {
			topic = getMessageVariable(pack.Message, k.topicVariable)
		}
		if k.hashVariable != nil {
			key = sarama.StringEncoder(getMessageVariable(pack.Message, k.hashVariable))
		}

		if msgBytes, err := or.Encode(pack); err == nil {
			if msgBytes != nil {
				err = k.producer.QueueMessage(topic, key, sarama.ByteEncoder(msgBytes))
				if err != nil {
					atomic.AddInt64(&k.processMessageFailures, 1)
					or.LogError(err)
				}
			} else {
				atomic.AddInt64(&k.processMessageDiscards, 1)
			}
		} else {
			atomic.AddInt64(&k.processMessageFailures, 1)
			or.LogError(err)
		}
		pack.Recycle()
	}
	errChan <- Shutdown
	wg.Wait()
	return
}
Exemplo n.º 13
0
func (this *SandboxManagerFilter) Run(fr pipeline.FilterRunner,
	h pipeline.PluginHelper) (err error) {

	inChan := fr.InChan()

	var ok = true
	var pack *pipeline.PipelinePack
	var delta int64

	this.restoreSandboxes(fr, h, this.workingDirectory)
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break
			}
			atomic.AddInt64(&this.processMessageCount, 1)
			delta = time.Now().UnixNano() - pack.Message.GetTimestamp()
			if math.Abs(float64(delta)) >= 5e9 {
				fr.LogError(fmt.Errorf("Discarded control message: %d seconds skew",
					delta/1e9))
				pack.Recycle()
				break
			}
			action, _ := pack.Message.GetFieldValue("action")
			switch action {
			case "load":
				current := int(atomic.LoadInt32(&this.currentFilters))
				if current < this.maxFilters {
					err := this.loadSandbox(fr, h, this.workingDirectory, pack.Message)
					if err != nil {
						fr.LogError(err)
					}
				} else {
					fr.LogError(fmt.Errorf("%s attempted to load more than %d filters",
						fr.Name(), this.maxFilters))
				}
			case "unload":
				fv, _ := pack.Message.GetFieldValue("name")
				if name, ok := fv.(string); ok {
					name = getSandboxName(fr.Name(), name)
					if this.pConfig.RemoveFilterRunner(name) {
						removeAll(this.workingDirectory, fmt.Sprintf("%s.*", name))
					}
				}
			}
			pack.Recycle()
		}
	}
	return
}
func (k *KinesisOutput) HandlePackage(or pipeline.OutputRunner, pack *pipeline.PipelinePack) error {

	// If we are flushing, wait until we have finished.
	k.flushLock.Lock()
	defer k.flushLock.Unlock()

	// encode the packages.
	msg, err := or.Encode(pack)
	if err != nil {
		errOut := fmt.Errorf("Error encoding message: %s", err)
		or.LogError(errOut)
		pack.Recycle(nil)
		return errOut
	}

	// If we only care about the Payload...
	if k.config.PayloadOnly {
		msg = []byte(pack.Message.GetPayload())
	}

	var tmp []byte
	// if we already have data then we should append.
	if len(k.batchedData) > 0 {
		tmp = append(append(k.batchedData, []byte(",")...), msg...)
	} else {
		tmp = msg
	}

	// if we can't fit the data in this record
	if len(tmp) > k.KINESIS_RECORD_SIZE {
		// add the existing data to the output batch
		array := append(append([]byte("["), k.batchedData...), []byte("]")...)
		k.AddToRecordBatch(or, array)

		// update the batched data to only contain the current message.
		k.batchedData = msg
	} else {
		// otherwise we add the existing data to a batch
		k.batchedData = tmp
	}

	// do reporting and tidy up
	atomic.AddInt64(&k.processMessageCount, 1)
	pack.Recycle(nil)

	return nil
}
Exemplo n.º 15
0
func (so *SentryOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	var (
		udpAddrStr string
		udpAddr    *net.UDPAddr
		socket     net.Conn
		e          error
		ok         bool
		pack       *pipeline.PipelinePack
	)

	sentryMsg := &SentryMsg{
		dataPacket: make([]byte, 0, so.config.MaxSentryBytes),
	}

	for pack = range or.InChan() {
		e = so.prepSentryMsg(pack, sentryMsg)
		pack.Recycle()
		if e != nil {
			or.LogError(e)
			continue
		}

		udpAddrStr = sentryMsg.parsedDsn.Host
		if socket, ok = so.udpMap[udpAddrStr]; !ok {
			if len(so.udpMap) > so.config.MaxUdpSockets {
				or.LogError(fmt.Errorf("Max # of UDP sockets [%d] reached.",
					so.config.MaxUdpSockets))
				continue
			}

			if udpAddr, e = net.ResolveUDPAddr("udp", udpAddrStr); e != nil {
				or.LogError(fmt.Errorf("can't resolve UDP address %s: %s",
					udpAddrStr, e))
				continue
			}

			if socket, e = net.DialUDP("udp", nil, udpAddr); e != nil {
				or.LogError(fmt.Errorf("can't dial UDP socket: %s", e))
				continue
			}
			so.udpMap[sentryMsg.parsedDsn.Host] = socket
		}
		socket.Write(sentryMsg.dataPacket)
	}
	return
}
Exemplo n.º 16
0
func (no *NsqOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	var (
		encoder client.Encoder
		msg     *message.Message
		msgBody []byte = make([]byte, 0, 1024)
		pack    *pipeline.PipelinePack
	)

	conf := no.conf
	encoder = client.NewProtobufEncoder(nil)

	for pack = range or.InChan() {
		if conf.Serialize {
			msg = pack.Message
			if err = encoder.EncodeMessageStream(msg, &msgBody); err != nil {
				or.LogError(err)
				err = nil
				pack.Recycle()
				continue
			}
			//err := no.nsqwriter.PublishAsync(conf.Topic, []byte(pack.Message.GetPayload()), nil)
			//err = no.nsqwriter.PublishAsync(conf.Topic, msgBody, nil)
			_, _, err = no.nsqwriter.Publish(conf.Topic, msgBody)
			if err != nil {
				or.LogError(fmt.Errorf("error in writer.PublishAsync"))
			}
			msgBody = msgBody[:0]
		} else {
			err = no.nsqwriter.PublishAsync(conf.Topic, []byte(pack.Message.GetPayload()), nil)
			if err != nil {
				or.LogError(fmt.Errorf("error in writer.PublishAsync"))
			}
		}
		pack.Recycle()
	}

	return
}
Exemplo n.º 17
0
func (k *KinesisOutput) Run(or pipeline.OutputRunner, helper pipeline.PluginHelper) error {
	var (
		pack   *pipeline.PipelinePack
		msg    []byte
		pk     string
		err    error
		params *kin.PutRecordInput
	)

	if or.Encoder() == nil {
		return fmt.Errorf("Encoder required.")
	}

	for pack = range or.InChan() {
		msg, err = or.Encode(pack)
		if err != nil {
			or.LogError(fmt.Errorf("Error encoding message: %s", err))
			pack.Recycle(nil)
			continue
		}
		pk = fmt.Sprintf("%d-%s", pack.Message.Timestamp, pack.Message.Hostname)
		if k.config.PayloadOnly {
			msg = []byte(pack.Message.GetPayload())
		}
		params = &kin.PutRecordInput{
			Data:         msg,
			PartitionKey: aws.String(pk),
			StreamName:   aws.String(k.config.Stream),
		}
		_, err = k.Client.PutRecord(params)
		if err != nil {
			or.LogError(fmt.Errorf("Error pushing message to Kinesis: %s", err))
			pack.Recycle(nil)
			continue
		}
		pack.Recycle(nil)
	}

	return nil
}
Exemplo n.º 18
0
func (k *KafkaOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	defer func() {
		k.producer.Close()
		k.client.Close()
	}()

	if or.Encoder() == nil {
		return errors.New("Encoder required.")
	}

	inChan := or.InChan()
	errChan := k.producer.Errors()
	pInChan := k.producer.Input()
	shutdownChan := make(chan struct{})
	var wg sync.WaitGroup
	wg.Add(1)
	go k.processKafkaErrors(or, errChan, shutdownChan, &wg)

	var (
		pack  *pipeline.PipelinePack
		topic = k.config.Topic
		key   sarama.Encoder
	)

	for pack = range inChan {
		atomic.AddInt64(&k.processMessageCount, 1)

		if k.topicVariable != nil {
			topic = getMessageVariable(pack.Message, k.topicVariable)
		}
		if k.hashVariable != nil {
			key = sarama.StringEncoder(getMessageVariable(pack.Message, k.hashVariable))
		}

		msgBytes, err := or.Encode(pack)
		if err != nil {
			atomic.AddInt64(&k.processMessageFailures, 1)
			or.LogError(err)
			// Don't retry encoding errors.
			or.UpdateCursor(pack.QueueCursor)
			pack.Recycle(nil)
			continue
		}
		if msgBytes == nil {
			atomic.AddInt64(&k.processMessageDiscards, 1)
			or.UpdateCursor(pack.QueueCursor)
			pack.Recycle(nil)
			continue
		}
		pMessage := &sarama.ProducerMessage{
			Topic: topic,
			Key:   key,
			Value: sarama.ByteEncoder(msgBytes),
		}
		pInChan <- pMessage
		pack.Recycle(nil)
	}

	close(shutdownChan)
	wg.Wait()
	return
}
Exemplo n.º 19
0
func (s *SandboxOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	var (
		pack      *pipeline.PipelinePack
		retval    int
		inChan    = or.InChan()
		duration  int64
		startTime time.Time
		ok        = true
		ticker    = or.Ticker()
	)

	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break
			}
			if s.sample {
				startTime = time.Now()
			}
			retval = s.sb.ProcessMessage(pack)
			if s.sample {
				duration = time.Since(startTime).Nanoseconds()
				s.reportLock.Lock()
				s.processMessageDuration += duration
				s.processMessageSamples++
				s.reportLock.Unlock()
			}
			s.sample = 0 == rand.Intn(s.sampleDenominator)
			pack.Recycle()

			if retval == 0 {
				atomic.AddInt64(&s.processMessageCount, 1)
			} else if retval < 0 {
				atomic.AddInt64(&s.processMessageFailures, 1)
				em := s.sb.LastError()
				if len(em) > 0 {
					or.LogError(errors.New(em))
				}
			} else {
				err = fmt.Errorf("FATAL: %s", s.sb.LastError())
				ok = false
			}

		case t := <-ticker:
			startTime = time.Now()
			if retval = s.sb.TimerEvent(t.UnixNano()); retval != 0 {
				err = fmt.Errorf("FATAL: %s", s.sb.LastError())
				ok = false
			}
			duration = time.Since(startTime).Nanoseconds()
			s.reportLock.Lock()
			s.timerEventDuration += duration
			s.timerEventSamples++
			s.reportLock.Unlock()
		}
	}

	s.reportLock.Lock()
	var destroyErr error
	if s.sbc.PreserveData {
		destroyErr = s.sb.Destroy(s.preservationFile)
	} else {
		destroyErr = s.sb.Destroy("")
	}
	if destroyErr != nil {
		err = destroyErr
	}

	s.sb = nil
	s.reportLock.Unlock()
	return
}
Exemplo n.º 20
0
func (output *NsqOutput) Run(runner pipeline.OutputRunner,
	helper pipeline.PluginHelper) (err error) {
	if runner.Encoder() == nil {
		return errors.New("Encoder required.")
	}

	var (
		pack     *pipeline.PipelinePack
		outgoing []byte
		msg      RetryMsg
	)

	output.runner = runner
	inChan := runner.InChan()
	ok := true

	defer output.cleanup()

	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				return nil
			}
			outgoing, err = output.runner.Encode(pack)
			if err != nil {
				runner.LogError(err)
			} else {
				err = output.sendMessage(outgoing)
				if err != nil {
					output.runner.LogError(err)
					err = output.retryHelper.Wait()
					if err != nil {
						return
					}
					// Create a retry msg, and requeue it
					msg := RetryMsg{Body: outgoing, retryChan: output.retryChan, maxCount: output.MaxMsgRetries}
					err = msg.Retry()
					if err != nil {
						output.runner.LogError(err)
					}
				} else {
					output.retryHelper.Reset()
				}
			}
			pack.Recycle()
		case msg, ok = <-output.retryChan:
			if !ok {
				return nil
			}
			err = output.sendMessage(msg.Body)
			if err != nil {
				output.runner.LogError(err)
				err = output.retryHelper.Wait()
				if err != nil {
					return
				}
				// requeue the message
				err = msg.Retry()
				if err != nil {
					output.runner.LogError(err)
				}
			} else {
				output.retryHelper.Reset()
			}
		}
	}

	return nil
}
Exemplo n.º 21
0
func (s *SandboxInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) (err error) {
	abortChan := s.pConfig.Globals.AbortChan()
	s.sb.InjectMessage(func(payload, payload_type, payload_name string) int {
		var pack *pipeline.PipelinePack
		select {
		case pack = <-ir.InChan():
		case <-abortChan:
			pack.Recycle(nil)
			return 5
		}
		if err := proto.Unmarshal([]byte(payload), pack.Message); err != nil {
			pack.Recycle(nil)
			return 1
		}
		if s.tz != time.UTC {
			const layout = "2006-01-02T15:04:05.999999999" // remove the incorrect UTC tz info
			t := time.Unix(0, pack.Message.GetTimestamp())
			t = t.In(time.UTC)
			ct, _ := time.ParseInLocation(layout, t.Format(layout), s.tz)
			pack.Message.SetTimestamp(ct.UnixNano())
		}
		if err := ir.Inject(pack); err != nil {
			pack.Recycle(nil)
			return 5
		}
		atomic.AddInt64(&s.processMessageCount, 1)
		atomic.AddInt64(&s.processMessageBytes, int64(len(payload)))
		return 0
	})

	ticker := ir.Ticker()

	ok := true
	for ok {
		retval := s.sb.ProcessMessage(nil)
		if retval <= 0 { // Sandbox is in polling mode
			if retval < 0 {
				atomic.AddInt64(&s.processMessageFailures, 1)
				em := s.sb.LastError()
				if len(em) > 0 {
					ir.LogError(errors.New(em))
				}
			}
			if ticker == nil {
				ir.LogMessage("single run completed")
				break
			}
			select { // block until stop or poll interval
			case _, ok = <-s.stopChan:
			case <-ticker:
			}
		} else { // Sandbox is shutting down
			em := s.sb.LastError()
			if !strings.HasSuffix(em, "shutting down") {
				ir.LogError(errors.New(em))
			}
			break
		}
	}

	return s.destroy()
}
Exemplo n.º 22
0
func (this *SandboxFilter) Run(fr pipeline.FilterRunner, h pipeline.PluginHelper) (err error) {
	inChan := fr.InChan()
	ticker := fr.Ticker()

	var (
		ok             = true
		terminated     = false
		sample         = true
		blocking       = false
		backpressure   = false
		pack           *pipeline.PipelinePack
		retval         int
		msgLoopCount   uint
		injectionCount uint
		startTime      time.Time
		slowDuration   int64 = int64(this.pConfig.Globals.MaxMsgProcessDuration)
		duration       int64
		capacity       = cap(inChan) - 1
	)

	// We assign to the return value of Run() for errors in the closure so that
	// the plugin runner can determine what caused the SandboxFilter to return.
	this.sb.InjectMessage(func(payload, payload_type, payload_name string) int {
		if injectionCount == 0 {
			err = pipeline.TerminatedError("exceeded InjectMessage count")
			return 2
		}
		injectionCount--
		pack := h.PipelinePack(msgLoopCount)
		if pack == nil {
			err = pipeline.TerminatedError(fmt.Sprintf("exceeded MaxMsgLoops = %d",
				this.pConfig.Globals.MaxMsgLoops))
			return 3
		}
		if len(payload_type) == 0 { // heka protobuf message
			hostname := pack.Message.GetHostname()
			err := proto.Unmarshal([]byte(payload), pack.Message)
			if err == nil {
				// do not allow filters to override the following
				pack.Message.SetType("heka.sandbox." + pack.Message.GetType())
				pack.Message.SetLogger(fr.Name())
				pack.Message.SetHostname(hostname)
			} else {
				return 1
			}
		} else {
			pack.Message.SetType("heka.sandbox-output")
			pack.Message.SetLogger(fr.Name())
			pack.Message.SetPayload(payload)
			ptype, _ := message.NewField("payload_type", payload_type, "file-extension")
			pack.Message.AddField(ptype)
			pname, _ := message.NewField("payload_name", payload_name, "")
			pack.Message.AddField(pname)
		}
		if !fr.Inject(pack) {
			return 4
		}
		atomic.AddInt64(&this.injectMessageCount, 1)
		return 0
	})

	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break
			}
			atomic.AddInt64(&this.processMessageCount, 1)
			injectionCount = this.pConfig.Globals.MaxMsgProcessInject
			msgLoopCount = pack.MsgLoopCount

			if this.manager != nil { // only check for backpressure on dynamic plugins
				// reading a channel length is generally fast ~1ns
				// we need to check the entire chain back to the router
				backpressure = len(inChan) >= capacity ||
					fr.MatchRunner().InChanLen() >= capacity ||
					len(h.PipelineConfig().Router().InChan()) >= capacity
			}

			// performing the timing is expensive ~40ns but if we are
			// backpressured we need a decent sample set before triggering
			// termination
			if sample ||
				(backpressure && this.processMessageSamples < int64(capacity)) ||
				this.sbc.Profile {
				startTime = time.Now()
				sample = true
			}
			retval = this.sb.ProcessMessage(pack)
			if sample {
				duration = time.Since(startTime).Nanoseconds()
				this.reportLock.Lock()
				this.processMessageDuration += duration
				this.processMessageSamples++
				if this.sbc.Profile {
					this.profileMessageDuration = this.processMessageDuration
					this.profileMessageSamples = this.processMessageSamples
					if this.profileMessageSamples == int64(capacity)*10 {
						this.sbc.Profile = false
						// reset the normal sampling so it isn't heavily skewed by the profile values
						// i.e. process messages fast during profiling and then switch to malicious code
						this.processMessageDuration = this.profileMessageDuration / this.profileMessageSamples
						this.processMessageSamples = 1
					}
				}
				this.reportLock.Unlock()
			}
			if retval <= 0 {
				if backpressure && this.processMessageSamples >= int64(capacity) {
					if this.processMessageDuration/this.processMessageSamples > slowDuration ||
						fr.MatchRunner().GetAvgDuration() > slowDuration/5 {
						terminated = true
						blocking = true
					}
				}
				if retval < 0 {
					atomic.AddInt64(&this.processMessageFailures, 1)
					em := this.sb.LastError()
					if len(em) > 0 {
						fr.LogError(errors.New(em))
					}
				}
				sample = 0 == rand.Intn(this.sampleDenominator)
			} else {
				terminated = true
			}
			pack.Recycle()

		case t := <-ticker:
			injectionCount = this.pConfig.Globals.MaxMsgTimerInject
			startTime = time.Now()
			if retval = this.sb.TimerEvent(t.UnixNano()); retval != 0 {
				terminated = true
			}
			duration = time.Since(startTime).Nanoseconds()
			this.reportLock.Lock()
			this.timerEventDuration += duration
			this.timerEventSamples++
			this.reportLock.Unlock()
		}

		if terminated {
			pack := h.PipelinePack(0)
			pack.Message.SetType("heka.sandbox-terminated")
			pack.Message.SetLogger(pipeline.HEKA_DAEMON)
			message.NewStringField(pack.Message, "plugin", fr.Name())
			if blocking {
				pack.Message.SetPayload("sandbox is running slowly and blocking the router")
				// no lock on the ProcessMessage variables here because there are no active writers
				message.NewInt64Field(pack.Message, "ProcessMessageCount", this.processMessageCount, "count")
				message.NewInt64Field(pack.Message, "ProcessMessageFailures", this.processMessageFailures, "count")
				message.NewInt64Field(pack.Message, "ProcessMessageSamples", this.processMessageSamples, "count")
				message.NewInt64Field(pack.Message, "ProcessMessageAvgDuration",
					this.processMessageDuration/this.processMessageSamples, "ns")
				message.NewInt64Field(pack.Message, "MatchAvgDuration", fr.MatchRunner().GetAvgDuration(), "ns")
				message.NewIntField(pack.Message, "FilterChanLength", len(inChan), "count")
				message.NewIntField(pack.Message, "MatchChanLength", fr.MatchRunner().InChanLen(), "count")
				message.NewIntField(pack.Message, "RouterChanLength", len(h.PipelineConfig().Router().InChan()), "count")
			} else {
				pack.Message.SetPayload(this.sb.LastError())
			}
			fr.Inject(pack)
			break
		}
	}

	if this.manager != nil {
		this.manager.PluginExited()
	}

	this.reportLock.Lock()
	var destroyErr error
	if this.sbc.PreserveData {
		destroyErr = this.sb.Destroy(this.preservationFile)
	} else {
		destroyErr = this.sb.Destroy("")
	}
	if destroyErr != nil {
		err = destroyErr
	}

	this.sb = nil
	this.reportLock.Unlock()
	return
}
Exemplo n.º 23
0
func (ni *NsqInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error {
	// Get the InputRunner's chan to receive empty PipelinePacks
	var pack *pipeline.PipelinePack
	var err error
	var dRunner pipeline.DecoderRunner
	var decoder pipeline.Decoder
	var ok bool
	var e error

	//pos := 0
	//output := make([]*Message, 2)
	packSupply := ir.InChan()

	if ni.conf.Decoder != "" {
		if dRunner, ok = h.DecoderRunner(ni.conf.Decoder); !ok {
			return fmt.Errorf("Decoder not found: %s", ni.conf.Decoder)
		}
		decoder = dRunner.Decoder()
	}

	err = ni.nsqReader.ConnectToLookupd(ni.conf.Address)
	if err != nil {
		ir.LogError(errors.New("ConnectToLookupd failed."))
	}

	header := &message.Header{}

	stopped := false
	//readLoop:
	for !stopped {
		//stopped = true
		select {
		case <-ni.stopChan:
			ir.LogError(errors.New("get ni.stopChan, set stopped=true"))
			stopped = true
		default:
			pack = <-packSupply
			m, ok1 := <-ni.handler.logChan
			if !ok1 {
				stopped = true
				break
			}

			if ni.conf.Serialize {
				if dRunner == nil {
					pack.Recycle()
					ir.LogError(errors.New("Serialize messages require a decoder."))
				}
				//header := &message.Header{}
				_, msgOk := findMessage(m.msg.Body, header, &(pack.MsgBytes))
				if msgOk {
					dRunner.InChan() <- pack
				} else {
					pack.Recycle()
					ir.LogError(errors.New("Can't find Heka message."))
				}
				header.Reset()
			} else {
				//ir.LogError(fmt.Errorf("message body: %s", m.msg.Body))
				pack.Message.SetType("nsq")
				pack.Message.SetPayload(string(m.msg.Body))
				pack.Message.SetTimestamp(time.Now().UnixNano())
				var packs []*pipeline.PipelinePack
				if decoder == nil {
					packs = []*pipeline.PipelinePack{pack}
				} else {
					packs, e = decoder.Decode(pack)
				}
				if packs != nil {
					for _, p := range packs {
						ir.Inject(p)
					}
				} else {
					if e != nil {
						ir.LogError(fmt.Errorf("Couldn't parse Nsq message: %s", m.msg.Body))
					}
					pack.Recycle()
				}
			}
			m.returnChannel <- &nsq.FinishedMessage{m.msg.Id, 0, true}
			/*
			   output[pos] = m
			   pos++
			   if pos == 2 {
			           for pos > 0 {
			                   pos--
			                   m1 := output[pos]
			                   m1.returnChannel <- &nsq.FinishedMessage{m1.msg.Id, 0, true}
			                   output[pos] = nil
			           }
			   }
			*/
		}
	}
	return nil
}
Exemplo n.º 24
0
func (s *SandboxOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	var (
		pack      *pipeline.PipelinePack
		retval    int
		inChan    = or.InChan()
		duration  int64
		startTime time.Time
		ok        = true
		ticker    = or.Ticker()
	)

	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break
			}
			if s.sample {
				startTime = time.Now()
			}
			retval = s.sb.ProcessMessage(pack)
			if s.sample {
				duration = time.Since(startTime).Nanoseconds()
				s.reportLock.Lock()
				s.processMessageDuration += duration
				s.processMessageSamples++
				s.reportLock.Unlock()
			}
			s.sample = 0 == rand.Intn(s.sampleDenominator)

			or.UpdateCursor(pack.QueueCursor) // TODO: support retries?
			if retval == 0 {
				atomic.AddInt64(&s.processMessageCount, 1)
				pack.Recycle(nil)
			} else if retval < 0 {
				atomic.AddInt64(&s.processMessageFailures, 1)
				var e error
				em := s.sb.LastError()
				if len(em) > 0 {
					e = errors.New(em)
				}
				pack.Recycle(e)
			} else {
				err = fmt.Errorf("FATAL: %s", s.sb.LastError())
				pack.Recycle(err)
				ok = false
			}

		case t := <-ticker:
			startTime = time.Now()
			if retval = s.sb.TimerEvent(t.UnixNano()); retval != 0 {
				err = fmt.Errorf("FATAL: %s", s.sb.LastError())
				ok = false
			}
			duration = time.Since(startTime).Nanoseconds()
			s.reportLock.Lock()
			s.timerEventDuration += duration
			s.timerEventSamples++
			s.reportLock.Unlock()
		}
	}

	if err == nil && s.sbc.TimerEventOnShutdown {
		if retval = s.sb.TimerEvent(time.Now().UnixNano()); retval != 0 {
			err = fmt.Errorf("FATAL: %s", s.sb.LastError())
		}
	}

	destroyErr := s.destroy()
	if destroyErr != nil {
		if err != nil {
			or.LogError(err)
		}
		err = destroyErr
	}
	return err
}