func (input *NsqInput) Run(runner pipeline.InputRunner, helper pipeline.PluginHelper) (err error) { var ( dRunner pipeline.DecoderRunner ok bool ) if input.DecoderName != "" { if dRunner, ok = helper.DecoderRunner(input.DecoderName, fmt.Sprintf("%s-%s", runner.Name(), input.DecoderName)); !ok { return fmt.Errorf("Decoder not found: %s", input.DecoderName) } input.decoderChan = dRunner.InChan() } input.runner = runner input.packSupply = runner.InChan() input.consumer.AddHandler(input) err = input.consumer.ConnectToNSQDs(input.NsqdAddrs) if err != nil { return err } err = input.consumer.ConnectToNSQLookupds(input.LookupdAddrs) if err != nil { return err } <-input.consumer.StoppedChan() return nil }
func (rpsi *RedisPubSubInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error { var ( dRunner pipeline.DecoderRunner decoder pipeline.Decoder pack *pipeline.PipelinePack e error ok bool ) // Get the InputRunner's chan to receive empty PipelinePacks packSupply := ir.InChan() if rpsi.conf.DecoderName != "" { if dRunner, ok = h.DecoderRunner(rpsi.conf.DecoderName, fmt.Sprintf("%s-%s", ir.Name(), rpsi.conf.DecoderName)); !ok { return fmt.Errorf("Decoder not found: %s", rpsi.conf.DecoderName) } decoder = dRunner.Decoder() } //Connect to the channel psc := redis.PubSubConn{Conn: rpsi.conn} psc.PSubscribe(rpsi.conf.Channel) for { switch n := psc.Receive().(type) { case redis.PMessage: // Grab an empty PipelinePack from the InputRunner pack = <-packSupply pack.Message.SetType("redis_pub_sub") pack.Message.SetLogger(n.Channel) pack.Message.SetPayload(string(n.Data)) pack.Message.SetTimestamp(time.Now().UnixNano()) var packs []*pipeline.PipelinePack if decoder == nil { packs = []*pipeline.PipelinePack{pack} } else { packs, e = decoder.Decode(pack) } if packs != nil { for _, p := range packs { ir.Inject(p) } } else { if e != nil { ir.LogError(fmt.Errorf("Couldn't parse Redis message: %s", n.Data)) } pack.Recycle(nil) } case redis.Subscription: ir.LogMessage(fmt.Sprintf("Subscription: %s %s %d\n", n.Kind, n.Channel, n.Count)) if n.Count == 0 { return errors.New("No channel to subscribe") } case error: fmt.Printf("error: %v\n", n) return n } } return nil }
func (input *FilePollingInput) Run(runner pipeline.InputRunner, helper pipeline.PluginHelper) error { var ( data []byte pack *pipeline.PipelinePack dRunner pipeline.DecoderRunner ok bool err error ) if input.DecoderName != "" { if dRunner, ok = helper.DecoderRunner(input.DecoderName, fmt.Sprintf("%s-%s", runner.Name(), input.DecoderName)); !ok { return fmt.Errorf("Decoder not found: %s", input.DecoderName) } input.decoderChan = dRunner.InChan() } input.runner = runner hostname := helper.PipelineConfig().Hostname() packSupply := runner.InChan() tickChan := runner.Ticker() for { select { case <-input.stop: return nil case <-tickChan: } data, err = ioutil.ReadFile(input.FilePath) if err != nil { runner.LogError(fmt.Errorf("Error reading file: %s", err)) continue } pack = <-packSupply pack.Message.SetUuid(uuid.NewRandom()) pack.Message.SetTimestamp(time.Now().UnixNano()) pack.Message.SetType("heka.file.polling") pack.Message.SetHostname(hostname) pack.Message.SetPayload(string(data)) if field, err := message.NewField("TickerInterval", int(input.TickerInterval), ""); err != nil { runner.LogError(err) } else { pack.Message.AddField(field) } if field, err := message.NewField("FilePath", input.FilePath, ""); err != nil { runner.LogError(err) } else { pack.Message.AddField(field) } input.sendPack(pack) } return nil }
func (rli *RedisInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error { fmt.Println("Addr", rli.conf.Address) fmt.Println("key", rli.conf.Key) fmt.Println("batch_count:", rli.conf.Batch_count) fmt.Println("decoder:", rli.conf.Decoder) var ( dRunner pipeline.DecoderRunner decoder pipeline.Decoder ok bool e error reply interface{} vals []string msg string ) if rli.conf.Decoder != "" { if dRunner, ok = h.DecoderRunner(rli.conf.Decoder, fmt.Sprintf("%s-%s", ir.Name(), rli.conf.Decoder)); !ok { return fmt.Errorf("Decoder not found: %s", rli.conf.Decoder) } decoder = dRunner.Decoder() } for { reply, e = rli.conn.Do("BLPOP", rli.conf.Key, "0") if e == nil { vals, e = redis.Strings(reply, nil) msg = vals[1] if e == nil { rli.InsertMessage(ir, decoder, msg) } } reply, e = rli.batchlpop.Do(rli.conn, rli.conf.Key, rli.conf.Batch_count) if e == nil { vals, e = redis.Strings(reply, nil) if e == nil { for _, msg = range vals { rli.InsertMessage(ir, decoder, msg) } } else { fmt.Printf("err: %v\n", e) } } else { fmt.Printf("type: %T, error: %v\n", reply, e) return e } } return nil }