func redisGet(redisServer *RedisInputServer, consumer *redismq.Consumer) error {
consumer.ResetWorking()
rateCounter := ratecounter.NewRateCounter(1 * time.Second)
for {
unacked := consumer.GetUnackedLength()
if unacked > 0 {
log.Printf("Requeued %d messages\n", unacked)
consumer.RequeueWorking()
}
packages, err := consumer.MultiGet(recvBuffer)
if err == nil {
numPackages := len(packages)
if numPackages > 0 {
rateCounter.Incr(int64(numPackages))
err = packages[numPackages-1].MultiAck()
if err != nil {
log.Println("Failed to ack", err)
}
}
for i := range packages {
var ev buffer.Event
payload := string(packages[i].Payload)
ev.Text = &payload
if redisServer.config.JsonDecode {
decoder := json.NewDecoder(strings.NewReader(payload))
decoder.UseNumber()
err = decoder.Decode(&ev.Fields)
if err != nil {
continue
}
}
redisServer.receiver.Send(&ev)
}
} else {
log.Printf("Error reading from Redis: %s, sleeping", err)
time.Sleep(2 * time.Second)
}
}
return nil
}
// read parses the compressed data frame
func (p *Parser) read() (uint32, error) {
var seq, count uint32
var k, v []byte
var err error
r, err := zlib.NewReader(p.Conn)
if err != nil {
return seq, err
}
defer r.Close()
// Decompress
buff := new(bytes.Buffer)
io.Copy(buff, r)
p.buffer = buff
b := make([]byte, 2)
for i := uint32(0); i < p.wlen; i++ {
n, err := buff.Read(b)
if err == io.EOF {
return seq, err
}
if n == 0 {
continue
}
switch string(b) {
case "2D": // window size
binary.Read(buff, binary.BigEndian, &seq)
binary.Read(buff, binary.BigEndian, &count)
var ev buffer.Event
fields := make(map[string]interface{})
fields["timestamp"] = time.Now().Format(time.RFC3339Nano)
for j := uint32(0); j < count; j++ {
if k, v, err = p.readKV(); err != nil {
return seq, err
}
fields[string(k)] = string(v)
}
ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"])
ev.Offset, _ = strconv.ParseInt(fields["offset"].(string), 10, 64)
ev.Line = uint64(seq)
t := fields["line"].(string)
ev.Text = &t
ev.Fields = &fields
// Send to the receiver which is a buffer. We block because...
p.Recv.Send(&ev)
case "2J": // JSON
//log.Printf("Got JSON data")
binary.Read(buff, binary.BigEndian, &seq)
binary.Read(buff, binary.BigEndian, &count)
jsonData := make([]byte, count)
_, err := p.buffer.Read(jsonData)
//log.Printf("Got message: %s", jsonData)
if err != nil {
return seq, err
}
var ev buffer.Event
var fields map[string]interface{}
decoder := json.NewDecoder(strings.NewReader(string(jsonData)))
decoder.UseNumber()
err = decoder.Decode(&fields)
if err != nil {
return seq, err
}
ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"])
jsonNumber := fields["offset"].(json.Number)
ev.Offset, _ = jsonNumber.Int64()
ev.Line = uint64(seq)
t := fields["message"].(string)
ev.Text = &t
ev.Fields = &fields
// Send to the receiver which is a buffer. We block because...
p.Recv.Send(&ev)
default:
return seq, fmt.Errorf("unknown type: %s", b)
}
}
return seq, nil
}