// FromReader parse the file
func FromReader(file io.Reader, listCommands *[]engine.Command) error {
var command engine.Command
pState := parserState{}
// Create our lexer
// NewSize(startState, reader, readerBufLen, channelCap)
lex := lexer.NewSize(lexFunc, file, 100, 1)
var lastTokenType = TokenNil
// Process lexer-emitted tokens
for t := lex.NextToken(); lexer.TokenTypeEOF != t.Type(); t = lex.NextToken() {
switch t.Type() {
case TokenWord:
if lastTokenType != TokenWord {
tokenValue := string(t.Bytes())
// TokenWord is a double or single quoted string?
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
if !pState.IsUsing && !pState.IsCommand && !pState.IsValue {
return errors.New("Invalid quoted string: " + tokenValue)
}
setQuotedString(tokenValue, &command, &pState)
// TokenWord is the Index name?
// using <TokenWord> ...
} else if pState.IsUsing {
indexDbParts := strings.Split(tokenValue, ".")
if len(indexDbParts) < 2 {
return fmt.Errorf("Invalid USING <index>.<database>: %s", tokenValue)
}
command.Index = indexDbParts[0]
command.Database = strings.Join(indexDbParts[1:], ".")
pState.IsUsing = false
// TokenWord is the key of command?
// using document.db mergeset <TokenWord> ...
} else if pState.IsCommand {
if strings.HasPrefix(tokenValue, "int(") {
pState.KVType = engine.TypeInt
pState.IsCastOpen = true
} else if strings.HasPrefix(tokenValue, "uint(") {
pState.KVType = engine.TypeUint
pState.IsCastOpen = true
} else if strings.HasPrefix(tokenValue, "float(") {
pState.KVType = engine.TypeFloat
pState.IsCastOpen = true
} else {
command.Key = []byte(tokenValue)
command.KeyType = engine.TypeString
pState.IsCommand = false
pState.IsValue = true
}
// TokenWord is the command value?
// using document.db mergeset name <TokenWord>
} else if pState.IsValue {
if tokenValue == ")" && pState.IsCastOpen {
pState.IsCastOpen = false
} else if strings.HasPrefix(tokenValue, "uint(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeUint
} else if strings.HasPrefix(tokenValue, "int(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeInt
} else if strings.HasPrefix(tokenValue, "float(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeFloat
} else {
command.Value = []byte(tokenValue)
command.ValueType = engine.TypeString
pState.IsValue = false
}
} else {
// Here we handle the available KEYWORDS
if pState.IsCastOpen && strings.HasPrefix(tokenValue, ")") {
pState.IsCastOpen = false
pState.KVType = 0
// Keyword USING
} else if !pState.IsUsing && strings.ToLower(tokenValue) == "using" {
pState.IsUsing = true
} else if !pState.IsCommand {
// Must be a command KEYWORD
// see commandsAvailable
if !isValidCommand(tokenValue) {
return fmt.Errorf("Invalid keyword '"+tokenValue+"': %s", command)
}
pState.IsCommand = true
pState.IsUsing = false
command.Command = strings.ToLower(tokenValue)
}
}
}
case TokenDoubleQuotedString:
if pState.IsDoubleQuotedString {
if pState.IsCommand {
pState.IsCommand = false
pState.IsValue = true
} else if pState.IsUsing {
pState.IsUsing = false
} else if pState.IsValue {
pState.IsValue = false
}
}
if pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
pState.IsDoubleQuotedString = !pState.IsDoubleQuotedString
}
case TokenSingleQuotedString:
if pState.IsSingleQuotedString {
if pState.IsCommand {
pState.IsCommand = false
pState.IsValue = true
} else if pState.IsUsing {
pState.IsUsing = false
} else if pState.IsValue {
pState.IsValue = false
}
}
if pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
pState.IsSingleQuotedString = !pState.IsSingleQuotedString
}
case TokenEscapedDoubleQuotedString:
if pState.IsSingleQuotedString {
panic("Escaped double quoted string inside single quoted string...")
} else if pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()[1:]), &command, &pState)
}
pState.IsEscapedDoubleQuotedString = !pState.IsEscapedDoubleQuotedString
case TokenEscapedSingleQuotedString:
if pState.IsDoubleQuotedString {
return errors.New("Escaped single quoted string inside double quoted string...")
} else if pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenSpace:
// Spaces only makes difference inside quotes
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenNewline:
// New lines only makes difference inside quotes
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenSemiColon:
if pState.IsSingleQuotedString || pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
*listCommands = append(*listCommands, command)
command = engine.Command{}
pState = parserState{}
}
case TokenNumbers:
tokenValue := string(t.Bytes())
if pState.IsSingleQuotedString || pState.IsDoubleQuotedString {
setQuotedString(tokenValue, &command, &pState)
} else if pState.IsUsing {
if index.ValidateIndexName(tokenValue) {
command.Index = tokenValue
pState.IsUsing = false
// TokenNumbers is the key of command?
// using document.db mergeset <TokenNumbers> ...
}
} else if pState.IsCommand {
var (
keyBytes []byte
keyType uint8
)
if strings.Contains(tokenValue, ".") {
tokenFloatValue, err := strconv.ParseFloat(tokenValue, 64)
if err != nil {
return fmt.Errorf("Failed to convert %s to float", tokenValue)
}
keyBytes = utils.Float64ToBytes(tokenFloatValue)
keyType = engine.TypeFloat
} else {
tokenIntValue, err := strconv.Atoi(tokenValue)
if err != nil {
return fmt.Errorf("Failed to convert %s to integer", tokenValue)
}
if pState.KVType == engine.TypeUint {
keyBytes = utils.Uint64ToBytes(uint64(tokenIntValue))
keyType = engine.TypeUint
} else if pState.KVType == engine.TypeFloat {
keyBytes = utils.Float64ToBytes(float64(tokenIntValue))
keyType = engine.TypeFloat
} else {
keyBytes = utils.Int64ToBytes(int64(tokenIntValue))
keyType = engine.TypeInt
}
}
command.Key = keyBytes
command.KeyType = keyType
pState.IsCommand = false
pState.IsValue = true
pState.KVType = 0
// TokenNumbers is the command value?
// using document.db mergeset name <TokenNumbers>
} else if pState.IsValue {
var (
valueBytes []byte
valueType uint8
tokenIntValue int64
tokenFloatValue float64
err error
)
if strings.Contains(tokenValue, ".") {
tokenFloatValue, err = strconv.ParseFloat(tokenValue, 64)
if err != nil {
return fmt.Errorf("Failed to convert %s to float", tokenValue)
}
valueBytes = utils.Float64ToBytes(tokenFloatValue)
valueType = engine.TypeFloat
} else {
tokenInt, err := strconv.Atoi(tokenValue)
if err != nil {
return fmt.Errorf("Failed to convert %s to integer", tokenValue)
}
tokenIntValue = int64(tokenInt)
valueBytes = utils.Int64ToBytes(int64(tokenIntValue))
valueType = engine.TypeInt
}
if command.Command == "mergeset" {
if valueType == engine.TypeFloat {
return fmt.Errorf("Failed to parse command. "+
"MergeSet value shall be a unsigned integer "+
"value: %v", tokenValue)
}
command.Value = utils.Uint64ToBytes(uint64(tokenIntValue))
command.ValueType = engine.TypeUint
} else {
command.Value = valueBytes
command.ValueType = valueType
}
pState.IsValue = false
}
default:
return errors.New("Failed to parse line at '" + string(t.Bytes()) + "'")
}
lastTokenType = t.Type()
}
// Checks if the last command was correctly parsed but
// doesn't have the semicolon at the end...
if validateCommand(command) {
*listCommands = append(*listCommands, command)
command = engine.Command{}
pState = parserState{}
// Checks if exists a invalid partial command
} else if command.Index != "" || command.Command != "" ||
command.Key != nil || command.Value != nil {
return fmt.Errorf("The last command wasn't correctly finished nor have the semicolon at end: %v", command)
}
return nil
}
func TestCliParserFromReader(t *testing.T) {
commands := []engine.Command{}
error := FromReader(strings.NewReader("using sample.TEST mergeset a 1;"), &commands)
if error != nil {
t.Error(error)
}
compareCommand(commands[0], engine.Command{
Index: "sample",
Database: "TEST",
Command: "mergeset",
Key: []byte("a"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
}, t)
compareArray(`using sample.test.idx mergeset a 2;
using sample.document.db set 1 "{id: 1, name: \"teste\"}";
using sample.lalala set hello "world";
using sample.mimimi get hello;
using sample.lelele delete "teste";
using sample.bleh.idx get uint(1);
using sample.aaaa.bbb set uint(10000) int(10);
using sample.bbbb.ccc mergeset "hellooooooooooooooooo" uint(102999299112211223);
using sample.aaa delete "bbb"
`, []engine.Command{
engine.Command{
Index: "sample",
Database: "test.idx",
Command: "mergeset",
Key: []byte("a"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
engine.Command{
Index: "sample",
Database: "document.db",
Command: "set",
Key: utils.Int64ToBytes(1),
KeyType: engine.TypeInt,
Value: []byte("{id: 1, name: \"teste\"}"),
ValueType: engine.TypeString,
},
engine.Command{
Index: "sample",
Database: "lalala",
Command: "set",
Key: []byte("hello"),
KeyType: engine.TypeString,
Value: []byte("world"),
ValueType: engine.TypeString,
},
engine.Command{
Index: "sample",
Database: "mimimi",
Command: "get",
Key: []byte("hello"),
KeyType: engine.TypeString,
},
engine.Command{
Index: "sample",
Database: "lelele",
Command: "delete",
Key: []byte("teste"),
KeyType: engine.TypeString,
},
engine.Command{
Index: "sample",
Database: "bleh.idx",
Command: "get",
Key: utils.Uint64ToBytes(1),
KeyType: engine.TypeUint,
},
engine.Command{
Index: "sample",
Database: "aaaa.bbb",
Command: "set",
Key: utils.Uint64ToBytes(10000),
KeyType: engine.TypeUint,
Value: utils.Int64ToBytes(10),
ValueType: engine.TypeInt,
},
engine.Command{
Index: "sample",
Database: "bbbb.ccc",
Command: "mergeset",
Key: []byte("hellooooooooooooooooo"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(102999299112211223),
ValueType: engine.TypeUint,
},
engine.Command{
Index: "sample",
Database: "aaa",
Command: "delete",
Key: []byte("bbb"),
KeyType: engine.TypeString,
},
}, t)
// underscore in the index name should pass
commands = []engine.Command{}
error = FromReader(strings.NewReader(`using sample.user_password set admin "s3cr3t"`),
&commands)
if error != nil {
t.Error(error)
}
compareCommand(commands[0], engine.Command{
Index: "sample",
Database: "user_password",
Command: "set",
Key: []byte("admin"),
KeyType: engine.TypeString,
Value: []byte("s3cr3t"),
ValueType: engine.TypeString,
}, t)
// invalid keyword "usinga"
shouldThrowError(`
usinga sample.test.idx set "hello" "world";
`, t)
}
func TestDateIndex(t *testing.T) {
var (
indexName = "document-sample-date"
indexDir = DataDirTmp + "/" + indexName
commands, expectedCommands []engine.Command
currentDate = time.Now()
currentDateStr = currentDate.Format(time.ANSIC)
expectedTime, _ = time.Parse(time.ANSIC, currentDateStr)
expectedNano = expectedTime.UnixNano()
docJSON = []byte(`{"id": 1, "createAt": "` + currentDateStr + `"}`)
err error
index *Index
metadata = Metadata{
"id": Metadata{
"type": "uint",
},
"createAt": Metadata{
"type": "date",
},
}
)
cfg := Config{
Debug: true,
DataDir: DataDirTmp,
}
err = os.MkdirAll(DataDirTmp, 0755)
if err != nil {
goto cleanup
}
index, err = New(indexName, cfg, true)
if err != nil {
t.Error(err)
goto cleanup
}
if _, err := os.Stat(indexDir); os.IsNotExist(err) {
t.Errorf("no such file or directory: %s", indexDir)
goto cleanup
}
commands, err = index.BuildAdd(1, docJSON, metadata)
if err != nil {
t.Error(err.Error())
goto cleanup
}
expectedCommands = []engine.Command{
{
Index: indexName,
Database: "document.db",
Key: utils.Uint64ToBytes(1),
KeyType: engine.TypeUint,
Value: docJSON,
ValueType: engine.TypeString,
Command: "set",
},
{
Index: indexName,
Database: "createat_int.idx",
Key: utils.Int64ToBytes(expectedNano),
KeyType: engine.TypeInt,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
{
Index: indexName,
Database: "id_uint.idx",
Key: utils.Uint64ToBytes(1),
KeyType: engine.TypeUint,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
}
if !compareCommands(t, commands, expectedCommands) {
goto cleanup
}
cleanup:
index.Close()
os.RemoveAll(indexDir)
}