Ejemplo n.º 1
0
func (p *parser) parseString(buf *streambuf.Buffer) (string, bool, bool) {
	line, err := buf.UntilCRLF()
	if err != nil {
		return "", true, false
	}

	if len(line) == 3 && line[1] == '-' && line[2] == '1' {
		return "nil", true, true
	}

	length, err := strconv.ParseInt(string(line[1:]), 10, 64)
	if err != nil {
		logp.Err("Failed to read bulk message: %s", err)
		return "", false, false
	}

	content, err := buf.CollectWithSuffix(int(length), []byte("\r\n"))
	if err != nil {
		if err != streambuf.ErrNoMoreBytes {
			return "", false, false
		}
		return "", true, false
	}

	return string(content), true, true
}
Ejemplo n.º 2
0
func (p *parser) dispatch(depth int, buf *streambuf.Buffer) (string, bool, bool, bool) {
	if buf.Len() == 0 {
		return "", false, true, false
	}

	var value string
	var iserror, ok, complete bool
	snapshot := buf.Snapshot()

	switch buf.Bytes()[0] {
	case '*':
		value, iserror, ok, complete = p.parseArray(depth, buf)
	case '$':
		value, ok, complete = p.parseString(buf)
	case ':':
		value, ok, complete = p.parseInt(buf)
	case '+':
		value, ok, complete = p.parseSimpleString(buf)
	case '-':
		iserror = true
		value, ok, complete = p.parseSimpleString(buf)
	default:
		debug("Unexpected message starting with %s", buf.Bytes()[0])
		return "", false, false, false
	}

	if !ok || !complete {
		buf.Restore(snapshot)
	}
	return value, iserror, ok, complete
}
Ejemplo n.º 3
0
func (p *parser) parseSimpleString(buf *streambuf.Buffer) (string, bool, bool) {
	line, err := buf.UntilCRLF()
	if err != nil {
		return "", true, false
	}

	return string(line[1:]), true, true
}
Ejemplo n.º 4
0
func (p *parser) parseArray(depth int, buf *streambuf.Buffer) (string, bool, bool, bool) {
	line, err := buf.UntilCRLF()
	if err != nil {
		debug("End of line not found, waiting for more data")
		return "", false, false, false
	}
	debug("line %s: %d", line, buf.BufferConsumed())

	if len(line) == 3 && line[1] == '-' && line[2] == '1' {
		return "nil", false, true, true
	}

	if len(line) == 2 && line[1] == '0' {
		return "[]", false, true, true
	}

	count, err := strconv.ParseInt(string(line[1:]), 10, 64)
	if err != nil {
		logp.Err("Failed to read number of bulk messages: %s", err)
		return "", false, false, false
	}
	if count < 0 {
		return "nil", false, true, true
	}

	content := make([]string, 0, count)
	// read sub elements

	iserror := false
	for i := 0; i < int(count); i++ {
		var value string
		var ok, complete bool

		value, iserror, ok, complete := p.dispatch(depth+1, buf)
		if !ok || !complete {
			debug("Array incomplete")
			return "", iserror, ok, complete
		}

		content = append(content, value)
	}

	if depth == 0 && isRedisCommand(content[0]) { // we've got a request
		p.message.IsRequest = true
		p.message.Method = content[0]
		p.message.Path = content[1]
	}

	var value string
	if depth == 0 && p.message.IsRequest {
		value = strings.Join(content, " ")
	} else {
		value = "[" + strings.Join(content, ", ") + "]"
	}
	return value, iserror, true, true
}
Ejemplo n.º 5
0
func withBinaryUint64(
	parser *parser,
	buf *streambuf.Buffer,
	f func(*message, uint64),
) error {
	val, err := buf.ReadNetUint64()
	if err == nil {
		f(parser.message, val)
	}
	return err
}
Ejemplo n.º 6
0
func withInt64Arg(
	parser *parser,
	buf *streambuf.Buffer,
	fn func(msg *message, v int64),
) error {
	parseNextArg(buf)
	value, err := buf.AsciiInt(false)
	if err == nil {
		fn(parser.message, value)
	}
	return textArgError(err)
}
Ejemplo n.º 7
0
func parseVersionNumber(parser *parser, buf *streambuf.Buffer) parseResult {
	msg := parser.message
	if msg.IsRequest {
		return parser.contWith(buf, parseStateDataBinary)
	}

	// size already checked
	bytes, _ := buf.Collect(int(msg.bytes))
	msg.str = memcacheString{bytes}

	return parser.yield(buf.BufferConsumed())
}
Ejemplo n.º 8
0
func doParseCommand(parser *parser, buf *streambuf.Buffer) parseResult {
	// check if binary + text command and dispatch
	if !buf.Avail(2) {
		return parser.needMore()
	}
	magic := buf.Bytes()[0]
	is_binary := magic == MemcacheMagicRequest || magic == MemcacheMagicResponse
	if is_binary {
		return parser.contWith(buf, parseStateBinaryCommand)
	} else {
		return parser.contWith(buf, parseStateTextCommand)
	}
}
Ejemplo n.º 9
0
func withUintArg(
	parser *parser,
	buf *streambuf.Buffer,
	fn func(msg *message, v uint32),
) error {
	msg := parser.message
	parseNextArg(buf)
	value, err := buf.AsciiUint(false)
	if err == nil {
		fn(msg, uint32(value))
	}
	return textArgError(err)
}
Ejemplo n.º 10
0
func (p *parser) parseInt(buf *streambuf.Buffer) (string, bool, bool) {
	line, err := buf.UntilCRLF()
	if err != nil {
		return "", true, false
	}

	number := string(line[1:])
	if _, err := strconv.ParseInt(number, 10, 64); err != nil {
		logp.Err("Failed to read integer reply: %s", err)
	}

	return number, true, true
}
Ejemplo n.º 11
0
func parseStatLine(parser *parser, hdr, buf *streambuf.Buffer) error {
	name, _ := parseStringArg(buf)
	value, _ := parseStringArg(buf)
	if buf.Failed() {
		return buf.Err()
	}

	msg := parser.message
	msg.stats = append(msg.stats, memcacheStat{
		memcacheString{name},
		memcacheString{value},
	})
	return nil
}
Ejemplo n.º 12
0
func doParseTextCommand(parser *parser, buf *streambuf.Buffer) parseResult {
	line, err := buf.UntilCRLF()
	if err != nil {
		if err == streambuf.ErrNoMoreBytes {
			return parser.needMore()
		}
		return parser.failing(err)
	}

	msg := parser.message
	command, args, err := splitCommandAndArgs(line)
	if err != nil {
		return parser.failing(err)
	}

	debug("parse command: '%s' '%s'", command, args)

	msg.IsRequest = 'a' <= command[0] && command[0] <= 'z'
	var cmd *textCommandType
	if msg.IsRequest {
		cmd = findTextCommandType(requestCommands, command)
	} else {
		cmd = findTextCommandType(responseCommands, command)
		if cmd == nil {
			b := command[0]
			if '0' <= b && b <= '9' {
				cmd = &counterResponse
			}
		}
	}
	if cmd == nil {
		debug("unknown command: %s", msg.command)
		if parser.config.parseUnkown {
			cmd = &unknownCommand
		} else {
			return parser.failing(ErrParserUnknownCommand)
		}
	}

	msg.command = &cmd.commandType
	msg.rawArgs = args
	msg.commandLine = memcacheString{line}
	msg.rawCommand = command

	// the command parser will work on already separated command line.
	// The parser will either yield a message directly, or switch to binary
	// data parsing mode, which is provided by explicit state
	return parser.contWithShallow(buf, cmd.parse)
}
Ejemplo n.º 13
0
func sockReadMessage(buf *streambuf.Buffer, in io.Reader) (*message, error) {
	buffer := make([]byte, 1024)
	for {
		n, err := in.Read(buffer)
		if err != nil {
			return nil, err
		}

		buf.Write(buffer[:n])
		msg, err := readMessage(buf)
		if msg != nil || err != nil {
			return msg, err
		}
	}
}
Ejemplo n.º 14
0
func parseNoReplyArg(buf *streambuf.Buffer) (bool, error) {
	debug("parse noreply")

	err := parseNextArg(buf)
	if err != nil {
		return false, textArgError(err)
	}

	var noreplyArg = []byte("noreply")
	noreply := bytes.HasPrefix(buf.Bytes(), noreplyArg)
	if !noreply {
		return false, ErrExpectedNoReply
	}
	return true, nil
}
Ejemplo n.º 15
0
func parseUdpHeader(buf *streambuf.Buffer) (mcUdpHeader, error) {
	var h mcUdpHeader
	h.requestId, _ = buf.ReadNetUint16()
	h.seqNumber, _ = buf.ReadNetUint16()
	h.numDatagrams, _ = buf.ReadNetUint16()
	buf.Advance(2) // ignore reserved
	return h, buf.Err()
}
Ejemplo n.º 16
0
func parseStatResponse(parser *parser, buf *streambuf.Buffer) parseResult {
	msg := parser.message
	if msg.IsRequest {
		return parser.contWith(buf, parseStateDataBinary)
	}

	bytes, _ := buf.Collect(int(msg.bytes))

	if len(msg.keys) == 0 {
		return parser.failing(ErrExpectedKeys)
	}

	msg.stats = append(msg.stats, memcacheStat{
		msg.keys[0],
		memcacheString{bytes},
	})
	return parser.yield(buf.BufferConsumed())
}
Ejemplo n.º 17
0
func parseDataBinary(parser *parser, buf *streambuf.Buffer) parseResult {
	msg := parser.message
	data, err := buf.Collect(int(msg.bytes - msg.bytesLost))
	if err != nil {
		if err == streambuf.ErrNoMoreBytes {
			return parser.needMore()
		}
		return parser.failing(err)
	}

	debug("found data message")
	if msg.bytesLost > 0 {
		msg.count_values++
	} else {
		parser.appendMessageData(data)
	}
	return parser.yield(buf.BufferConsumed() + int(msg.bytesLost))
}
Ejemplo n.º 18
0
func sockReadMessage(buf *streambuf.Buffer, in io.Reader) (*message, error) {
	for {
		// try parse message from buffered data
		msg, err := readMessage(buf)
		if msg != nil || (err != nil && err != streambuf.ErrNoMoreBytes) {
			return msg, err
		}

		// read next bytes from socket if incomplete message in buffer
		buffer := make([]byte, 1024)
		n, err := in.Read(buffer)
		if err != nil {
			return nil, err
		}

		buf.Write(buffer[:n])
	}
}
Ejemplo n.º 19
0
func parseData(parser *parser, buf *streambuf.Buffer) parseResult {
	msg := parser.message
	debug("parse message data (%v)", msg.bytes)
	data, err := buf.CollectWithSuffix(
		int(msg.bytes-msg.bytesLost),
		[]byte("\r\n"),
	)
	if err != nil {
		if err == streambuf.ErrNoMoreBytes {
			return parser.needMore()
		}
		return parser.failing(err)
	}

	debug("found message data")
	if msg.bytesLost > 0 {
		msg.count_values++
	} else {
		parser.appendMessageData(data)
	}
	return parser.yield(buf.BufferConsumed() + int(msg.bytesLost))
}
Ejemplo n.º 20
0
func parseNextArg(buf *streambuf.Buffer) error {
	err := buf.IgnoreSymbol(' ')
	if err == streambuf.ErrUnexpectedEOB || err == streambuf.ErrNoMoreBytes {
		buf.SetError(nil)
		return ErrNoMoreArgument
	}
	if buf.Len() == 0 {
		return ErrNoMoreArgument
	}
	return nil
}
Ejemplo n.º 21
0
func (parser *parser) parse(buf *streambuf.Buffer) (bool, bool) {
	snapshot := buf.Snapshot()

	content, iserror, ok, complete := parser.dispatch(0, buf)
	if !ok || !complete {
		// on error or incomplete message drop all parsing progress, due to
		// parse not being statefull among multiple calls
		// => parser needs to restart parsing all content
		buf.Restore(snapshot)
		return ok, complete
	}

	parser.message.IsError = iserror
	parser.message.Size = buf.BufferConsumed()
	parser.message.Message = content
	return true, true
}
Ejemplo n.º 22
0
func parseBinaryCounterResponse(
	parser *parser,
	buf *streambuf.Buffer,
) parseResult {
	msg := parser.message
	if msg.IsRequest {
		return parser.contWith(buf, parseStateDataBinary)
	}

	// size already checked
	bytes, _ := buf.Collect(int(msg.bytes))
	tmp := streambuf.NewFixed(bytes)
	err := withBinaryUint64(parser, tmp, func(msg *message, value uint64) {
		msg.value = value
	})
	if err != nil {
		return parser.failing(err)
	}

	buf.Advance(8)
	return parser.yield(buf.BufferConsumed())
}
Ejemplo n.º 23
0
func (p *parser) yieldNoData(buf *streambuf.Buffer) parseResult {
	return p.yield(buf.BufferConsumed())
}
Ejemplo n.º 24
0
func readMessage(buf *streambuf.Buffer) (*message, error) {
	if !buf.Avail(2) {
		return nil, nil
	}

	version, _ := buf.ReadNetUint8At(0)
	if version != '2' {
		return nil, errors.New("version error")
	}

	code, _ := buf.ReadNetUint8At(1)
	switch code {
	case 'W':
		if !buf.Avail(6) {
			return nil, nil
		}
		size, _ := buf.ReadNetUint32At(2)
		buf.Advance(6)
		buf.Reset()
		return &message{code: code, size: size}, buf.Err()
	case 'C':
		if !buf.Avail(6) {
			return nil, nil
		}
		len, _ := buf.ReadNetUint32At(2)
		if !buf.Avail(int(len) + 6) {
			return nil, nil
		}
		buf.Advance(6)

		tmp, _ := buf.Collect(int(len))
		buf.Reset()

		dataBuf := streambuf.New(nil)
		// decompress data
		decomp, err := zlib.NewReader(streambuf.NewFixed(tmp))
		if err != nil {
			return nil, err
		}
		// dataBuf.ReadFrom(streambuf.NewFixed(tmp))
		dataBuf.ReadFrom(decomp)
		decomp.Close()

		// unpack data
		dataBuf.Fix()
		var events []*message
		for dataBuf.Len() > 0 {
			version, _ := dataBuf.ReadNetUint8()
			if version != '2' {
				return nil, errors.New("version error 2")
			}

			code, _ := dataBuf.ReadNetUint8()
			if code != 'J' {
				return nil, errors.New("expected json data frame")
			}

			seq, _ := dataBuf.ReadNetUint32()
			payloadLen, _ := dataBuf.ReadNetUint32()
			jsonRaw, _ := dataBuf.Collect(int(payloadLen))

			var doc interface{}
			err = json.Unmarshal(jsonRaw, &doc)
			if err != nil {
				return nil, err
			}

			events = append(events, &message{
				code: code,
				seq:  seq,
				doc:  doc.(map[string]interface{}),
			})
		}
		return &message{code: 'C', events: events}, nil
	default:
		return nil, errors.New("unknown code")
	}
}
Ejemplo n.º 25
0
func parseBinaryCommand(parser *parser, buf *streambuf.Buffer) parseResult {
	debug("on binary message")

	if !buf.Avail(memcacheHeaderSize) {
		return parser.needMore()
	}

	msg := parser.message
	msg.isBinary = true
	magic, _ := buf.ReadNetUint8At(0)
	switch magic {
	case MemcacheMagicRequest:
		msg.IsRequest = true
	case MemcacheMagicResponse:
		msg.IsRequest = false
	default:
		return parser.failing(ErrInvalidMemcacheMagic)
	}

	opcode, _ := buf.ReadNetUint8At(1)
	keyLen, err := buf.ReadNetUint16At(2)
	extraLen, _ := buf.ReadNetUint8At(4)
	if err != nil {
		return parser.failing(err)
	}

	debug("magic: %v", magic)
	debug("opcode: %v", opcode)
	debug("extra len: %v", extraLen)
	debug("key len: %v", keyLen)

	totalHeaderLen := memcacheHeaderSize + int(extraLen) + int(keyLen)
	debug("total header len: %v", totalHeaderLen)
	if !buf.Avail(totalHeaderLen) {
		return parser.needMore()
	}

	command := memcacheBinaryCommandTable[memcacheOpcode(opcode)]
	if command == nil {
		debug("unknown command")
		command = binaryUnknownCommand
	}

	msg.opcode = memcacheOpcode(opcode)
	msg.command = command
	msg.isQuiet = isQuietOpcode(msg.opcode)
	return parser.contWithShallow(buf, command.parse)
}
Ejemplo n.º 26
0
func readMessage(buf *streambuf.Buffer) (*message, error) {
	if !buf.Avail(2) {
		return nil, nil
	}

	version, _ := buf.ReadNetUint8At(0)
	if version != '1' {
		return nil, errors.New("version error")
	}

	code, _ := buf.ReadNetUint8At(1)
	switch code {
	case 'W':
		if !buf.Avail(6) {
			return nil, nil
		}
		size, _ := buf.ReadNetUint32At(2)
		buf.Advance(6)
		buf.Reset()
		return &message{code: code, size: size}, buf.Err()
	case 'C':
		if !buf.Avail(6) {
			return nil, nil
		}
		len, _ := buf.ReadNetUint32At(2)
		if !buf.Avail(int(len) + 6) {
			return nil, nil
		}
		buf.Advance(6)

		tmp, _ := buf.Collect(int(len))
		buf.Reset()

		dataBuf := streambuf.New(nil)
		// decompress data
		decomp, err := zlib.NewReader(streambuf.NewFixed(tmp))
		if err != nil {
			return nil, err
		}
		// dataBuf.ReadFrom(streambuf.NewFixed(tmp))
		dataBuf.ReadFrom(decomp)
		decomp.Close()

		// unpack data
		dataBuf.Fix()
		var events []*message
		for dataBuf.Len() > 0 {
			version, _ := dataBuf.ReadNetUint8()
			if version != '1' {
				return nil, errors.New("version error 2")
			}

			code, _ := dataBuf.ReadNetUint8()
			if code != 'D' {
				return nil, errors.New("expected data frame")
			}

			seq, _ := dataBuf.ReadNetUint32()
			pairCount, _ := dataBuf.ReadNetUint32()
			kv := make(map[string]string)
			for i := 0; i < int(pairCount); i++ {
				keyLen, _ := dataBuf.ReadNetUint32()
				keyRaw, _ := dataBuf.Collect(int(keyLen))
				valLen, _ := dataBuf.ReadNetUint32()
				valRaw, _ := dataBuf.Collect(int(valLen))
				kv[string(keyRaw)] = string(valRaw)
			}

			events = append(events, &message{code: code, seq: seq, kv: kv})
		}
		return &message{code: 'C', events: events}, nil
	default:
		return nil, errors.New("unknown code")
	}
}
Ejemplo n.º 27
0
func parseStringArg(buf *streambuf.Buffer) ([]byte, error) {
	if err := parseNextArg(buf); err != nil {
		return nil, err
	}
	return buf.UntilSymbol(' ', false)
}