Example #1
0
func redisGet(redisServer *RedisInputServer, consumer *redismq.Consumer) error {
	consumer.ResetWorking()
	rateCounter := ratecounter.NewRateCounter(1 * time.Second)

	for {
		unacked := consumer.GetUnackedLength()

		if unacked > 0 {
			log.Printf("Requeued %d messages\n", unacked)
			consumer.RequeueWorking()
		}

		packages, err := consumer.MultiGet(recvBuffer)

		if err == nil {
			numPackages := len(packages)

			if numPackages > 0 {
				rateCounter.Incr(int64(numPackages))
				err = packages[numPackages-1].MultiAck()

				if err != nil {
					log.Println("Failed to ack", err)
				}
			}

			for i := range packages {
				var ev buffer.Event
				payload := string(packages[i].Payload)
				ev.Text = &payload

				if redisServer.config.JsonDecode {
					decoder := json.NewDecoder(strings.NewReader(payload))
					decoder.UseNumber()

					err = decoder.Decode(&ev.Fields)

					if err != nil {
						continue
					}
				}

				redisServer.receiver.Send(&ev)
			}
		} else {
			log.Printf("Error reading from Redis: %s, sleeping", err)
			time.Sleep(2 * time.Second)
		}
	}

	return nil
}
Example #2
0
// read parses the compressed data frame
func (p *Parser) read() (uint32, error) {
	var seq, count uint32
	var k, v []byte
	var err error

	r, err := zlib.NewReader(p.Conn)
	if err != nil {
		return seq, err
	}
	defer r.Close()

	// Decompress
	buff := new(bytes.Buffer)
	io.Copy(buff, r)
	p.buffer = buff

	b := make([]byte, 2)
	for i := uint32(0); i < p.wlen; i++ {
		n, err := buff.Read(b)
		if err == io.EOF {
			return seq, err
		}

		if n == 0 {
			continue
		}

		switch string(b) {
		case "2D": // window size
			binary.Read(buff, binary.BigEndian, &seq)
			binary.Read(buff, binary.BigEndian, &count)

			var ev buffer.Event
			fields := make(map[string]interface{})
			fields["timestamp"] = time.Now().Format(time.RFC3339Nano)

			for j := uint32(0); j < count; j++ {
				if k, v, err = p.readKV(); err != nil {
					return seq, err
				}
				fields[string(k)] = string(v)
			}

			ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"])
			ev.Offset, _ = strconv.ParseInt(fields["offset"].(string), 10, 64)
			ev.Line = uint64(seq)
			t := fields["line"].(string)
			ev.Text = &t
			ev.Fields = &fields

			// Send to the receiver which is a buffer. We block because...
			p.Recv.Send(&ev)
		case "2J": // JSON
			//log.Printf("Got JSON data")
			binary.Read(buff, binary.BigEndian, &seq)
			binary.Read(buff, binary.BigEndian, &count)
			jsonData := make([]byte, count)
			_, err := p.buffer.Read(jsonData)
			//log.Printf("Got message: %s", jsonData)

			if err != nil {
				return seq, err
			}

			var ev buffer.Event
			var fields map[string]interface{}
			decoder := json.NewDecoder(strings.NewReader(string(jsonData)))
			decoder.UseNumber()
			err = decoder.Decode(&fields)

			if err != nil {
				return seq, err
			}
			ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"])
			jsonNumber := fields["offset"].(json.Number)
			ev.Offset, _ = jsonNumber.Int64()
			ev.Line = uint64(seq)
			t := fields["message"].(string)
			ev.Text = &t
			ev.Fields = &fields

			// Send to the receiver which is a buffer. We block because...
			p.Recv.Send(&ev)

		default:
			return seq, fmt.Errorf("unknown type: %s", b)
		}
	}

	return seq, nil
}