func (p *parser) parseString(buf *streambuf.Buffer) (string, bool, bool) {
line, err := buf.UntilCRLF()
if err != nil {
return "", true, false
}
if len(line) == 3 && line[1] == '-' && line[2] == '1' {
return "nil", true, true
}
length, err := strconv.ParseInt(string(line[1:]), 10, 64)
if err != nil {
logp.Err("Failed to read bulk message: %s", err)
return "", false, false
}
content, err := buf.CollectWithSuffix(int(length), []byte("\r\n"))
if err != nil {
if err != streambuf.ErrNoMoreBytes {
return "", false, false
}
return "", true, false
}
return string(content), true, true
}
func (p *parser) parseString(buf *streambuf.Buffer) (common.NetString, bool, bool) {
line, err := buf.UntilCRLF()
if err != nil {
return empty, true, false
}
if len(line) == 3 && line[1] == '-' && line[2] == '1' {
return nilStr, true, true
}
length, err := parseInt(line[1:])
if err != nil {
logp.Err("Failed to read bulk message: %s", err)
return empty, false, false
}
content, err := buf.CollectWithSuffix(int(length), []byte("\r\n"))
if err != nil {
if err != streambuf.ErrNoMoreBytes {
return common.NetString{}, false, false
}
return common.NetString{}, true, false
}
return common.NetString(content), true, true
}
func (p *parser) dispatch(depth int, buf *streambuf.Buffer) (common.NetString, bool, bool, bool) {
if buf.Len() == 0 {
return empty, false, true, false
}
var value common.NetString
var iserror, ok, complete bool
snapshot := buf.Snapshot()
switch buf.Bytes()[0] {
case '*':
value, iserror, ok, complete = p.parseArray(depth, buf)
case '$':
value, ok, complete = p.parseString(buf)
case ':':
value, ok, complete = p.parseInt(buf)
case '+':
value, ok, complete = p.parseSimpleString(buf)
case '-':
iserror = true
value, ok, complete = p.parseSimpleString(buf)
default:
if isDebug {
debugf("Unexpected message starting with %s", buf.Bytes()[0])
}
return empty, false, false, false
}
if !ok || !complete {
buf.Restore(snapshot)
}
return value, iserror, ok, complete
}
func (p *parser) parseSimpleString(buf *streambuf.Buffer) (common.NetString, bool, bool) {
line, err := buf.UntilCRLF()
if err != nil {
return empty, true, false
}
return common.NetString(line[1:]), true, true
}
func (p *parser) parseArray(depth int, buf *streambuf.Buffer) (string, bool, bool, bool) {
line, err := buf.UntilCRLF()
if err != nil {
debug("End of line not found, waiting for more data")
return "", false, false, false
}
debug("line %s: %d", line, buf.BufferConsumed())
if len(line) == 3 && line[1] == '-' && line[2] == '1' {
return "nil", false, true, true
}
if len(line) == 2 && line[1] == '0' {
return "[]", false, true, true
}
count, err := strconv.ParseInt(string(line[1:]), 10, 64)
if err != nil {
logp.Err("Failed to read number of bulk messages: %s", err)
return "", false, false, false
}
if count < 0 {
return "nil", false, true, true
}
content := make([]string, 0, count)
// read sub elements
iserror := false
for i := 0; i < int(count); i++ {
var value string
var ok, complete bool
value, iserror, ok, complete := p.dispatch(depth+1, buf)
if !ok || !complete {
debug("Array incomplete")
return "", iserror, ok, complete
}
content = append(content, value)
}
if depth == 0 && isRedisCommand(content[0]) { // we've got a request
p.message.IsRequest = true
p.message.Method = content[0]
p.message.Path = content[1]
}
var value string
if depth == 0 && p.message.IsRequest {
value = strings.Join(content, " ")
} else {
value = "[" + strings.Join(content, ", ") + "]"
}
return value, iserror, true, true
}
func withBinaryUint64(
parser *parser,
buf *streambuf.Buffer,
f func(*message, uint64),
) error {
val, err := buf.ReadNetUint64()
if err == nil {
f(parser.message, val)
}
return err
}
func parseVersionNumber(parser *parser, buf *streambuf.Buffer) parseResult {
msg := parser.message
if msg.IsRequest {
return parser.contWith(buf, parseStateDataBinary)
}
// size already checked
bytes, _ := buf.Collect(int(msg.bytes))
msg.str = memcacheString{bytes}
return parser.yield(buf.BufferConsumed())
}
func doParseCommand(parser *parser, buf *streambuf.Buffer) parseResult {
// check if binary + text command and dispatch
if !buf.Avail(2) {
return parser.needMore()
}
magic := buf.Bytes()[0]
isBinary := magic == memcacheMagicRequest || magic == memcacheMagicResponse
if isBinary {
return parser.contWith(buf, parseStateBinaryCommand)
}
return parser.contWith(buf, parseStateTextCommand)
}
func withInt64Arg(
parser *parser,
buf *streambuf.Buffer,
fn func(msg *message, v int64),
) error {
parseNextArg(buf)
value, err := buf.IntASCII(false)
if err == nil {
fn(parser.message, value)
}
return textArgError(err)
}
func (p *parser) parseInt(buf *streambuf.Buffer) (string, bool, bool) {
line, err := buf.UntilCRLF()
if err != nil {
return "", true, false
}
number := string(line[1:])
if _, err := strconv.ParseInt(number, 10, 64); err != nil {
logp.Err("Failed to read integer reply: %s", err)
}
return number, true, true
}
func withUintArg(
parser *parser,
buf *streambuf.Buffer,
fn func(msg *message, v uint32),
) error {
msg := parser.message
parseNextArg(buf)
value, err := buf.UintASCII(false)
if err == nil {
fn(msg, uint32(value))
}
return textArgError(err)
}
func parseStatLine(parser *parser, hdr, buf *streambuf.Buffer) error {
name, _ := parseStringArg(buf)
value, _ := parseStringArg(buf)
if buf.Failed() {
return buf.Err()
}
msg := parser.message
msg.stats = append(msg.stats, memcacheStat{
memcacheString{name},
memcacheString{value},
})
return nil
}
func doParseTextCommand(parser *parser, buf *streambuf.Buffer) parseResult {
line, err := buf.UntilCRLF()
if err != nil {
if err == streambuf.ErrNoMoreBytes {
return parser.needMore()
}
return parser.failing(err)
}
msg := parser.message
command, args, err := splitCommandAndArgs(line)
if err != nil {
return parser.failing(err)
}
debug("parse command: '%s' '%s'", command, args)
msg.IsRequest = 'a' <= command[0] && command[0] <= 'z'
var cmd *textCommandType
if msg.IsRequest {
cmd = findTextCommandType(requestCommands, command)
} else {
cmd = findTextCommandType(responseCommands, command)
if cmd == nil {
b := command[0]
if '0' <= b && b <= '9' {
cmd = &counterResponse
}
}
}
if cmd == nil {
debug("unknown command: %s", msg.command)
if parser.config.parseUnkown {
cmd = &unknownCommand
} else {
return parser.failing(ErrParserUnknownCommand)
}
}
msg.command = &cmd.commandType
msg.rawArgs = args
msg.commandLine = memcacheString{line}
msg.rawCommand = command
// the command parser will work on already separated command line.
// The parser will either yield a message directly, or switch to binary
// data parsing mode, which is provided by explicit state
return parser.contWithShallow(buf, cmd.parse)
}
func parseUdpHeader(buf *streambuf.Buffer) (mcUdpHeader, error) {
var h mcUdpHeader
h.requestId, _ = buf.ReadNetUint16()
h.seqNumber, _ = buf.ReadNetUint16()
h.numDatagrams, _ = buf.ReadNetUint16()
buf.Advance(2) // ignore reserved
return h, buf.Err()
}
func parseNoReplyArg(buf *streambuf.Buffer) (bool, error) {
debug("parse noreply")
err := parseNextArg(buf)
if err != nil {
return false, textArgError(err)
}
var noreplyArg = []byte("noreply")
noreply := bytes.HasPrefix(buf.Bytes(), noreplyArg)
if !noreply {
return false, ErrExpectedNoReply
}
return true, nil
}
func parseDataBinary(parser *parser, buf *streambuf.Buffer) parseResult {
msg := parser.message
data, err := buf.Collect(int(msg.bytes - msg.bytesLost))
if err != nil {
if err == streambuf.ErrNoMoreBytes {
return parser.needMore()
}
return parser.failing(err)
}
debug("found data message")
if msg.bytesLost > 0 {
msg.count_values++
} else {
parser.appendMessageData(data)
}
return parser.yield(buf.BufferConsumed() + int(msg.bytesLost))
}
func sockReadMessage(buf *streambuf.Buffer, in io.Reader) (*message, error) {
for {
// try parse message from buffered data
msg, err := readMessage(buf)
if msg != nil || (err != nil && err != streambuf.ErrNoMoreBytes) {
return msg, err
}
// read next bytes from socket if incomplete message in buffer
buffer := make([]byte, 1024)
n, err := in.Read(buffer)
if err != nil {
return nil, err
}
buf.Write(buffer[:n])
}
}
func parseStatResponse(parser *parser, buf *streambuf.Buffer) parseResult {
msg := parser.message
if msg.IsRequest {
return parser.contWith(buf, parseStateDataBinary)
}
bytes, _ := buf.Collect(int(msg.bytes))
if len(msg.keys) == 0 {
return parser.failing(ErrExpectedKeys)
}
msg.stats = append(msg.stats, memcacheStat{
msg.keys[0],
memcacheString{bytes},
})
return parser.yield(buf.BufferConsumed())
}
func parseData(parser *parser, buf *streambuf.Buffer) parseResult {
msg := parser.message
debug("parse message data (%v)", msg.bytes)
data, err := buf.CollectWithSuffix(
int(msg.bytes-msg.bytesLost),
[]byte("\r\n"),
)
if err != nil {
if err == streambuf.ErrNoMoreBytes {
return parser.needMore()
}
return parser.failing(err)
}
debug("found message data")
if msg.bytesLost > 0 {
msg.count_values++
} else {
parser.appendMessageData(data)
}
return parser.yield(buf.BufferConsumed() + int(msg.bytesLost))
}
func parseNextArg(buf *streambuf.Buffer) error {
err := buf.IgnoreSymbol(' ')
if err == streambuf.ErrUnexpectedEOB || err == streambuf.ErrNoMoreBytes {
buf.SetError(nil)
return ErrNoMoreArgument
}
if buf.Len() == 0 {
return ErrNoMoreArgument
}
return nil
}
func readMessageType(buf *streambuf.Buffer) (byte, error) {
if !buf.Avail(2) {
return 0, nil
}
version, _ := buf.ReadNetUint8At(0)
if version != '2' {
return 0, errors.New("version error")
}
code, _ := buf.ReadNetUint8At(1)
return code, nil
}
func (p *parser) parse(buf *streambuf.Buffer) (bool, bool) {
snapshot := buf.Snapshot()
content, iserror, ok, complete := p.dispatch(0, buf)
if !ok || !complete {
// on error or incomplete message drop all parsing progress, due to
// parse not being statefull among multiple calls
// => parser needs to restart parsing all content
buf.Restore(snapshot)
return ok, complete
}
p.message.isError = iserror
p.message.size = buf.BufferConsumed()
p.message.message = content
return true, true
}
func parseBinaryCounterResponse(
parser *parser,
buf *streambuf.Buffer,
) parseResult {
msg := parser.message
if msg.IsRequest {
return parser.contWith(buf, parseStateDataBinary)
}
// size already checked
bytes, _ := buf.Collect(int(msg.bytes))
tmp := streambuf.NewFixed(bytes)
err := withBinaryUint64(parser, tmp, func(msg *message, value uint64) {
msg.value = value
})
if err != nil {
return parser.failing(err)
}
buf.Advance(8)
return parser.yield(buf.BufferConsumed())
}
func parseBinaryCommand(parser *parser, buf *streambuf.Buffer) parseResult {
debug("on binary message")
if !buf.Avail(memcacheHeaderSize) {
return parser.needMore()
}
msg := parser.message
msg.isBinary = true
magic, _ := buf.ReadNetUint8At(0)
switch magic {
case MemcacheMagicRequest:
msg.IsRequest = true
case MemcacheMagicResponse:
msg.IsRequest = false
default:
return parser.failing(ErrInvalidMemcacheMagic)
}
opcode, _ := buf.ReadNetUint8At(1)
keyLen, err := buf.ReadNetUint16At(2)
extraLen, _ := buf.ReadNetUint8At(4)
if err != nil {
return parser.failing(err)
}
debug("magic: %v", magic)
debug("opcode: %v", opcode)
debug("extra len: %v", extraLen)
debug("key len: %v", keyLen)
totalHeaderLen := memcacheHeaderSize + int(extraLen) + int(keyLen)
debug("total header len: %v", totalHeaderLen)
if !buf.Avail(totalHeaderLen) {
return parser.needMore()
}
command := memcacheBinaryCommandTable[memcacheOpcode(opcode)]
if command == nil {
debug("unknown command")
command = binaryUnknownCommand
}
msg.opcode = memcacheOpcode(opcode)
msg.command = command
msg.isQuiet = isQuietOpcode(msg.opcode)
return parser.contWithShallow(buf, command.parse)
}
func parseStringArg(buf *streambuf.Buffer) ([]byte, error) {
if err := parseNextArg(buf); err != nil {
return nil, err
}
return buf.UntilSymbol(' ', false)
}
func readWindowSize(buf *streambuf.Buffer) (uint32, error) {
return buf.ReadNetUint32At(2)
}
func (p *parser) parseArray(depth int, buf *streambuf.Buffer) (common.NetString, bool, bool, bool) {
line, err := buf.UntilCRLF()
if err != nil {
if isDebug {
debugf("End of line not found, waiting for more data")
}
return empty, false, false, false
}
if isDebug {
debugf("line %s: %d", line, buf.BufferConsumed())
}
if len(line) == 3 && line[1] == '-' && line[2] == '1' {
return nilStr, false, true, true
}
if len(line) == 2 && line[1] == '0' {
return emptyArr, false, true, true
}
count, err := parseInt(line[1:])
if err != nil {
logp.Err("Failed to read number of bulk messages: %s", err)
return empty, false, false, false
}
if count < 0 {
return nilStr, false, true, true
} else if count == 0 {
// should not happen, but handle just in case ParseInt did return 0
return emptyArr, false, true, true
}
// invariant: count > 0
// try to allocate content array right on stack
var content [][]byte
const arrayBufferSize = 32
if int(count) <= arrayBufferSize {
var arrayBuffer [arrayBufferSize][]byte
content = arrayBuffer[:0]
} else {
content = make([][]byte, 0, count)
}
contentLen := 0
// read sub elements
iserror := false
for i := 0; i < int(count); i++ {
var value common.NetString
var ok, complete bool
value, iserror, ok, complete := p.dispatch(depth+1, buf)
if !ok || !complete {
if isDebug {
debugf("Array incomplete")
}
return empty, iserror, ok, complete
}
content = append(content, []byte(value))
contentLen += len(value)
}
// handle top-level request command
if depth == 0 && isRedisCommand(content[0]) {
p.message.isRequest = true
p.message.method = content[0]
if len(content) > 1 {
p.message.path = content[1]
}
var value common.NetString
if contentLen > 1 {
tmp := make([]byte, contentLen+(len(content)-1)*1)
join(tmp, content, []byte(" "))
value = common.NetString(tmp)
} else {
value = common.NetString(content[0])
}
return value, iserror, true, true
}
// return redis array: [a, b, c]
tmp := make([]byte, 2+contentLen+(len(content)-1)*2)
tmp[0] = '['
join(tmp[1:], content, []byte(", "))
tmp[len(tmp)-1] = ']'
value := common.NetString(tmp)
return value, iserror, true, true
}
func readMessage(buf *streambuf.Buffer) (*message, error) {
if !buf.Avail(2) {
return nil, nil
}
version, _ := buf.ReadNetUint8At(0)
if version != '2' {
return nil, errors.New("version error")
}
code, _ := buf.ReadNetUint8At(1)
switch code {
case 'W':
if !buf.Avail(6) {
return nil, nil
}
size, _ := buf.ReadNetUint32At(2)
buf.Advance(6)
buf.Reset()
return &message{code: code, size: size}, buf.Err()
case 'C':
if !buf.Avail(6) {
return nil, nil
}
len, _ := buf.ReadNetUint32At(2)
if !buf.Avail(int(len) + 6) {
return nil, nil
}
buf.Advance(6)
tmp, _ := buf.Collect(int(len))
buf.Reset()
dataBuf := streambuf.New(nil)
// decompress data
decomp, err := zlib.NewReader(streambuf.NewFixed(tmp))
if err != nil {
return nil, err
}
// dataBuf.ReadFrom(streambuf.NewFixed(tmp))
dataBuf.ReadFrom(decomp)
decomp.Close()
// unpack data
dataBuf.Fix()
var events []*message
for dataBuf.Len() > 0 {
version, _ := dataBuf.ReadNetUint8()
if version != '2' {
return nil, errors.New("version error 2")
}
code, _ := dataBuf.ReadNetUint8()
if code != 'J' {
return nil, errors.New("expected json data frame")
}
seq, _ := dataBuf.ReadNetUint32()
payloadLen, _ := dataBuf.ReadNetUint32()
jsonRaw, _ := dataBuf.Collect(int(payloadLen))
var doc interface{}
err = json.Unmarshal(jsonRaw, &doc)
if err != nil {
return nil, err
}
events = append(events, &message{
code: code,
seq: seq,
doc: doc.(map[string]interface{}),
})
}
return &message{code: 'C', events: events}, nil
default:
return nil, errors.New("unknown code")
}
}
func (p *parser) yieldNoData(buf *streambuf.Buffer) parseResult {
return p.yield(buf.BufferConsumed())
}