Exemple #1
0
func (self *KafkaInput) Run(runner plugins.InputRunner) (err error) {
	counter := fmt.Sprintf("Tag:%s,Type:%s", self.common.Tag, self.common.Type)
	mc := metrics.NewCounter(counter)

	for {
		msg, err := self.consumer.Consume()
		if err != nil && err != kafka.ErrNoData {
			log.Printf("Consume :%s", err)
			break
		}
		pack := <-runner.InChan()
		pack.MsgBytes = bytes.TrimSpace(msg.Value)
		pack.Msg.Tag = self.common.Tag
		pack.Msg.Timestamp = time.Now().Unix()
		mc.Add(1)
		runner.RouterChan() <- pack

	}
	return nil
}
Exemple #2
0
func (this *TailInput) Run(runner plugins.InputRunner) (err error) {
	defer func() {

		if err := recover(); err != nil {
			log.Fatalln("recover panic at err:", err)
		}
		if this.checkpointFile != nil {
			this.checkpointFile.Close()
		}
	}()
	var seek int
	if this.config.OffsetValue > 0 {
		seek = os.SEEK_SET
	} else {
		seek = os.SEEK_END
	}
	t, err := tail.TailFile(this.config.Path, tail.Config{
		Poll:      true,
		ReOpen:    true,
		Follow:    true,
		MustExist: false,
		Location:  &tail.SeekInfo{int64(this.config.OffsetValue), seek},
	})
	if err != nil {
		return err
	}
	tick := time.NewTicker(time.Second * time.Duration(this.config.SyncInterval))
	count := 0

	for {
		select {
		case <-tick.C:
			{
				if count > 0 {
					offset, err := t.Tell()
					if err != nil {
						log.Println("Tell return error: ", err)
						continue
					}
					if err = this.writeCheckpoint(offset); err != nil {
						return err
					}

					count = 0
				}
			}
		case line := <-t.Lines:
			{
				pack := <-runner.InChan()
				pack.MsgBytes = []byte(line.Text)
				pack.Msg.Tag = this.common.Tag
				pack.Msg.Timestamp = time.Now().Unix()
				count++
				runner.RouterChan() <- pack
			}
		}
	}
	err = t.Wait()
	return err

}