示例#1
0
func (self *UdpInput) Run(runner plugins.InputRunner) error {
	var (
		e error
	)
	self.runner = runner
	buf := make([]byte, UDP_PACKET_SIZE)
	stopped := false
	counter := fmt.Sprintf("Tag:%s,Type:%s", self.common.Tag, self.common.Type)
	mc := metrics.NewCounter(counter)
	msgbytes := make([]byte, 0, 10000)

	for !stopped {
		select {
		case <-self.stopChan:
			stopped = true
		default:
			n, _, err := self.listener.ReadFromUDP(buf)
			if err != nil {
				log.Printf("read from udp err : %s", err)
				continue
			}
			//log.Printf("get %d from %s: %s", n, addr, buf[0:n])
			self.buffer.Write(buf[:n])
			for true {
				line, err := self.buffer.ReadBytes('\n')
				if err != nil {
					break
				}
				if !bytes.HasSuffix(line, []byte{'\n'}) {
					msgbytes = append(msgbytes, line...)
					continue
				} else {
					msgbytes = append(msgbytes, line...)
					pack := <-self.runner.InChan()
					pack.MsgBytes = bytes.TrimSpace(msgbytes)
					pack.Msg.Tag = self.common.Tag
					pack.Msg.Timestamp = time.Now().Unix()
					mc.Add(1)
					msgbytes = msgbytes[:0]

					self.runner.RouterChan() <- pack
				}
			}

		}
	}
	return e
}
示例#2
0
func (self *KafkaInput) Run(runner plugins.InputRunner) (err error) {
	counter := fmt.Sprintf("Tag:%s,Type:%s", self.common.Tag, self.common.Type)
	mc := metrics.NewCounter(counter)

	for {
		msg, err := self.consumer.Consume()
		if err != nil && err != kafka.ErrNoData {
			log.Printf("Consume :%s", err)
			break
		}
		pack := <-runner.InChan()
		pack.MsgBytes = bytes.TrimSpace(msg.Value)
		pack.Msg.Tag = self.common.Tag
		pack.Msg.Timestamp = time.Now().Unix()
		mc.Add(1)
		runner.RouterChan() <- pack

	}
	return nil
}
示例#3
0
// Listen on the provided TCP connection, extracting messages from the incoming
// data until the connection is closed or Stop is called on the input.
func (self *TcpInput) handleConnection(conn net.Conn) {
	var (
		frag []byte
		err  error = nil
		pack *plugins.PipelinePack
	)
	//raddr := conn.RemoteAddr().String()
	//host, _, err := net.SplitHostPort(raddr)
	//if err != nil {
	//	host = raddr
	//}
	//log.Printf("handle conn: %s, host: %s", raddr, host)
	counter := fmt.Sprintf("Tag:%s,Type:%s", self.common.Tag, self.common.Type)
	mc := metrics.NewCounter(counter)
	defer func() {
		conn.Close()
		self.wg.Done()
	}()

	buf := make([]byte, 1024)
	b1 := []byte{}
	count := 0
	limit_run_times := 60
	stopped := false
	reader := bufio.NewReaderSize(conn, 8192)

	ticker := time.NewTicker(time.Duration(1) * time.Minute)
	defer ticker.Stop()
	for !stopped {
		conn.SetReadDeadline(time.Now().Add(5 * time.Second))
		select {
		case <-self.stopChan:
			stopped = true
		case <-ticker.C:
			if count == 0 || limit_run_times <= 0 {
				//log.Printf("remove unused conn : %s", raddr)
				stopped = true
			}
			limit_run_times -= 1
			count = 0
		default:
			frag, err = reader.ReadSlice('\n')
			if err != nil {
				//log.Printf("disconnect : %s", raddr)
				stopped = true
			}

			if len(frag) == 0 {
				continue
			}
			buf = append(b1, frag...)
			count++
			pack = <-self.runner.InChan()
			pack.MsgBytes = bytes.TrimSpace(buf[:])
			pack.Msg.Tag = self.common.Tag
			pack.Msg.Timestamp = time.Now().Unix()
			mc.Add(1)
			self.runner.RouterChan() <- pack
			buf = buf[:0]
		}
	}
	buf = nil
	reader = nil
}