func (self *UdpInput) Run(runner plugins.InputRunner) error {
var (
e error
)
self.runner = runner
buf := make([]byte, UDP_PACKET_SIZE)
stopped := false
counter := fmt.Sprintf("Tag:%s,Type:%s", self.common.Tag, self.common.Type)
mc := metrics.NewCounter(counter)
msgbytes := make([]byte, 0, 10000)
for !stopped {
select {
case <-self.stopChan:
stopped = true
default:
n, _, err := self.listener.ReadFromUDP(buf)
if err != nil {
log.Printf("read from udp err : %s", err)
continue
}
//log.Printf("get %d from %s: %s", n, addr, buf[0:n])
self.buffer.Write(buf[:n])
for true {
line, err := self.buffer.ReadBytes('\n')
if err != nil {
break
}
if !bytes.HasSuffix(line, []byte{'\n'}) {
msgbytes = append(msgbytes, line...)
continue
} else {
msgbytes = append(msgbytes, line...)
pack := <-self.runner.InChan()
pack.MsgBytes = bytes.TrimSpace(msgbytes)
pack.Msg.Tag = self.common.Tag
pack.Msg.Timestamp = time.Now().Unix()
mc.Add(1)
msgbytes = msgbytes[:0]
self.runner.RouterChan() <- pack
}
}
}
}
return e
}
func (self *KafkaInput) Run(runner plugins.InputRunner) (err error) {
counter := fmt.Sprintf("Tag:%s,Type:%s", self.common.Tag, self.common.Type)
mc := metrics.NewCounter(counter)
for {
msg, err := self.consumer.Consume()
if err != nil && err != kafka.ErrNoData {
log.Printf("Consume :%s", err)
break
}
pack := <-runner.InChan()
pack.MsgBytes = bytes.TrimSpace(msg.Value)
pack.Msg.Tag = self.common.Tag
pack.Msg.Timestamp = time.Now().Unix()
mc.Add(1)
runner.RouterChan() <- pack
}
return nil
}
// Listen on the provided TCP connection, extracting messages from the incoming
// data until the connection is closed or Stop is called on the input.
func (self *TcpInput) handleConnection(conn net.Conn) {
var (
frag []byte
err error = nil
pack *plugins.PipelinePack
)
//raddr := conn.RemoteAddr().String()
//host, _, err := net.SplitHostPort(raddr)
//if err != nil {
// host = raddr
//}
//log.Printf("handle conn: %s, host: %s", raddr, host)
counter := fmt.Sprintf("Tag:%s,Type:%s", self.common.Tag, self.common.Type)
mc := metrics.NewCounter(counter)
defer func() {
conn.Close()
self.wg.Done()
}()
buf := make([]byte, 1024)
b1 := []byte{}
count := 0
limit_run_times := 60
stopped := false
reader := bufio.NewReaderSize(conn, 8192)
ticker := time.NewTicker(time.Duration(1) * time.Minute)
defer ticker.Stop()
for !stopped {
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
select {
case <-self.stopChan:
stopped = true
case <-ticker.C:
if count == 0 || limit_run_times <= 0 {
//log.Printf("remove unused conn : %s", raddr)
stopped = true
}
limit_run_times -= 1
count = 0
default:
frag, err = reader.ReadSlice('\n')
if err != nil {
//log.Printf("disconnect : %s", raddr)
stopped = true
}
if len(frag) == 0 {
continue
}
buf = append(b1, frag...)
count++
pack = <-self.runner.InChan()
pack.MsgBytes = bytes.TrimSpace(buf[:])
pack.Msg.Tag = self.common.Tag
pack.Msg.Timestamp = time.Now().Unix()
mc.Add(1)
self.runner.RouterChan() <- pack
buf = buf[:0]
}
}
buf = nil
reader = nil
}