func (self *KafkaInput) Run(runner plugins.InputRunner) (err error) {
counter := fmt.Sprintf("Tag:%s,Type:%s", self.common.Tag, self.common.Type)
mc := metrics.NewCounter(counter)
for {
msg, err := self.consumer.Consume()
if err != nil && err != kafka.ErrNoData {
log.Printf("Consume :%s", err)
break
}
pack := <-runner.InChan()
pack.MsgBytes = bytes.TrimSpace(msg.Value)
pack.Msg.Tag = self.common.Tag
pack.Msg.Timestamp = time.Now().Unix()
mc.Add(1)
runner.RouterChan() <- pack
}
return nil
}
func (this *TailInput) Run(runner plugins.InputRunner) (err error) {
defer func() {
if err := recover(); err != nil {
log.Fatalln("recover panic at err:", err)
}
if this.checkpointFile != nil {
this.checkpointFile.Close()
}
}()
var seek int
if this.config.OffsetValue > 0 {
seek = os.SEEK_SET
} else {
seek = os.SEEK_END
}
t, err := tail.TailFile(this.config.Path, tail.Config{
Poll: true,
ReOpen: true,
Follow: true,
MustExist: false,
Location: &tail.SeekInfo{int64(this.config.OffsetValue), seek},
})
if err != nil {
return err
}
tick := time.NewTicker(time.Second * time.Duration(this.config.SyncInterval))
count := 0
for {
select {
case <-tick.C:
{
if count > 0 {
offset, err := t.Tell()
if err != nil {
log.Println("Tell return error: ", err)
continue
}
if err = this.writeCheckpoint(offset); err != nil {
return err
}
count = 0
}
}
case line := <-t.Lines:
{
pack := <-runner.InChan()
pack.MsgBytes = []byte(line.Text)
pack.Msg.Tag = this.common.Tag
pack.Msg.Timestamp = time.Now().Unix()
count++
runner.RouterChan() <- pack
}
}
}
err = t.Wait()
return err
}