func TestBuildAddDocument(t *testing.T) {
var (
indexName = "document-sample"
commands, expectedCommands []engine.Command
docJSON = []byte(`{"id": 1}`)
err error
index *Index
indexDir = DataDirTmp + "/" + indexName
)
index, err = createIndex(indexName, t)
if err != nil {
t.Error(err)
return
}
commands, err = index.BuildAdd(1, docJSON, nil)
if err != nil {
t.Error(err.Error())
goto cleanup
}
expectedCommands = []engine.Command{
{
Index: indexName,
Database: "document.db",
Key: utils.Uint64ToBytes(1),
KeyType: engine.TypeUint,
Value: docJSON,
ValueType: engine.TypeString,
Command: "set",
},
{
Index: indexName,
Database: "id_float.idx",
Key: utils.Float64ToBytes(1),
KeyType: engine.TypeFloat,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
}
if !compareCommands(t, commands, expectedCommands) {
goto cleanup
}
docJSON = []byte(`{
"title": "NeoSearch - Reverse Index",
"description": "Neoway Full Text Search"
}`)
expectedCommands = []engine.Command{
{
Index: indexName,
Database: "document.db",
Command: "set",
Key: utils.Uint64ToBytes(2),
KeyType: engine.TypeUint,
Value: docJSON,
ValueType: engine.TypeString,
},
{
Index: indexName,
Database: "description_string.idx",
Command: "mergeset",
Key: []byte("neoway"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "description_string.idx",
Command: "mergeset",
Key: []byte("full"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "description_string.idx",
Command: "mergeset",
Key: []byte("text"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "description_string.idx",
Command: "mergeset",
Key: []byte("search"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "description_string.idx",
Command: "mergeset",
Key: []byte("neoway full text search"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "title_string.idx",
Command: "mergeset",
Key: []byte("neosearch"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "title_string.idx",
Command: "mergeset",
Key: []byte("-"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "title_string.idx",
Command: "mergeset",
Key: []byte("reverse"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "title_string.idx",
Command: "mergeset",
Key: []byte("index"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
{
Index: indexName,
Database: "title_string.idx",
Command: "mergeset",
Key: []byte("neosearch - reverse index"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(2),
ValueType: engine.TypeUint,
},
}
commands, err = index.BuildAdd(2, docJSON, nil)
if err != nil {
t.Error(err)
goto cleanup
}
if !compareCommands(t, commands, expectedCommands) {
goto cleanup
}
cleanup:
index.Close()
os.RemoveAll(indexDir)
}
func TestBuildAddObjectDocument(t *testing.T) {
var (
indexName = "document-with-object-sample"
indexDir = DataDirTmp + "/" + indexName
commands, expectedCommands []engine.Command
docJSON = []byte(`
{
"id": 1,
"address": {
"city": "florianópolis",
"district": "Itacorubi",
"street": "Patricio Farias",
"latlon": [
-27.545198,
-48.504827
]
}
}`)
err error
index *Index
)
cfg := Config{
Debug: false,
DataDir: DataDirTmp,
}
err = os.MkdirAll(DataDirTmp, 0755)
if err != nil {
goto cleanup
}
index, err = New(indexName, cfg, true)
if err != nil {
t.Error(err)
goto cleanup
}
if _, err := os.Stat(indexDir); os.IsNotExist(err) {
t.Errorf("no such file or directory: %s", indexDir)
goto cleanup
}
commands, err = index.BuildAdd(1, docJSON, nil)
if err != nil {
t.Error(err.Error())
goto cleanup
}
expectedCommands = []engine.Command{
{
Index: indexName,
Database: "document.db",
Key: utils.Uint64ToBytes(1),
KeyType: engine.TypeUint,
Value: docJSON,
ValueType: engine.TypeString,
Command: "set",
},
{
Index: indexName,
Database: "address.city_string.idx",
Key: []byte("florianópolis"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
{
Index: indexName,
Database: "address.district_string.idx",
Key: []byte("itacorubi"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
{
Index: indexName,
Database: "address.latlon_float.idx",
Key: utils.Float64ToBytes(-27.545198),
KeyType: engine.TypeFloat,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
{
Index: indexName,
Database: "address.latlon_float.idx",
Key: utils.Float64ToBytes(-48.504827),
KeyType: engine.TypeFloat,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
{
Index: indexName,
Database: "address.street_string.idx",
Key: []byte("patricio"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
{
Index: indexName,
Database: "address.street_string.idx",
Key: []byte("farias"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
{
Index: indexName,
Database: "address.street_string.idx",
Key: []byte("patricio farias"),
KeyType: engine.TypeString,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
{
Index: indexName,
Database: "id_float.idx",
Key: utils.Float64ToBytes(1),
KeyType: engine.TypeFloat,
Value: utils.Uint64ToBytes(1),
ValueType: engine.TypeUint,
Command: "mergeset",
},
}
if !compareCommands(t, commands, expectedCommands) {
goto cleanup
}
cleanup:
index.Close()
os.RemoveAll(indexDir)
}
// FromReader parse the file
func FromReader(file io.Reader, listCommands *[]engine.Command) error {
var command engine.Command
pState := parserState{}
// Create our lexer
// NewSize(startState, reader, readerBufLen, channelCap)
lex := lexer.NewSize(lexFunc, file, 100, 1)
var lastTokenType = TokenNil
// Process lexer-emitted tokens
for t := lex.NextToken(); lexer.TokenTypeEOF != t.Type(); t = lex.NextToken() {
switch t.Type() {
case TokenWord:
if lastTokenType != TokenWord {
tokenValue := string(t.Bytes())
// TokenWord is a double or single quoted string?
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
if !pState.IsUsing && !pState.IsCommand && !pState.IsValue {
return errors.New("Invalid quoted string: " + tokenValue)
}
setQuotedString(tokenValue, &command, &pState)
// TokenWord is the Index name?
// using <TokenWord> ...
} else if pState.IsUsing {
indexDbParts := strings.Split(tokenValue, ".")
if len(indexDbParts) < 2 {
return fmt.Errorf("Invalid USING <index>.<database>: %s", tokenValue)
}
command.Index = indexDbParts[0]
command.Database = strings.Join(indexDbParts[1:], ".")
pState.IsUsing = false
// TokenWord is the key of command?
// using document.db mergeset <TokenWord> ...
} else if pState.IsCommand {
if strings.HasPrefix(tokenValue, "int(") {
pState.KVType = engine.TypeInt
pState.IsCastOpen = true
} else if strings.HasPrefix(tokenValue, "uint(") {
pState.KVType = engine.TypeUint
pState.IsCastOpen = true
} else if strings.HasPrefix(tokenValue, "float(") {
pState.KVType = engine.TypeFloat
pState.IsCastOpen = true
} else {
command.Key = []byte(tokenValue)
command.KeyType = engine.TypeString
pState.IsCommand = false
pState.IsValue = true
}
// TokenWord is the command value?
// using document.db mergeset name <TokenWord>
} else if pState.IsValue {
if tokenValue == ")" && pState.IsCastOpen {
pState.IsCastOpen = false
} else if strings.HasPrefix(tokenValue, "uint(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeUint
} else if strings.HasPrefix(tokenValue, "int(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeInt
} else if strings.HasPrefix(tokenValue, "float(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeFloat
} else {
command.Value = []byte(tokenValue)
command.ValueType = engine.TypeString
pState.IsValue = false
}
} else {
// Here we handle the available KEYWORDS
if pState.IsCastOpen && strings.HasPrefix(tokenValue, ")") {
pState.IsCastOpen = false
pState.KVType = 0
// Keyword USING
} else if !pState.IsUsing && strings.ToLower(tokenValue) == "using" {
pState.IsUsing = true
} else if !pState.IsCommand {
// Must be a command KEYWORD
// see commandsAvailable
if !isValidCommand(tokenValue) {
return fmt.Errorf("Invalid keyword '"+tokenValue+"': %s", command)
}
pState.IsCommand = true
pState.IsUsing = false
command.Command = strings.ToLower(tokenValue)
}
}
}
case TokenDoubleQuotedString:
if pState.IsDoubleQuotedString {
if pState.IsCommand {
pState.IsCommand = false
pState.IsValue = true
} else if pState.IsUsing {
pState.IsUsing = false
} else if pState.IsValue {
pState.IsValue = false
}
}
if pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
pState.IsDoubleQuotedString = !pState.IsDoubleQuotedString
}
case TokenSingleQuotedString:
if pState.IsSingleQuotedString {
if pState.IsCommand {
pState.IsCommand = false
pState.IsValue = true
} else if pState.IsUsing {
pState.IsUsing = false
} else if pState.IsValue {
pState.IsValue = false
}
}
if pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
pState.IsSingleQuotedString = !pState.IsSingleQuotedString
}
case TokenEscapedDoubleQuotedString:
if pState.IsSingleQuotedString {
panic("Escaped double quoted string inside single quoted string...")
} else if pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()[1:]), &command, &pState)
}
pState.IsEscapedDoubleQuotedString = !pState.IsEscapedDoubleQuotedString
case TokenEscapedSingleQuotedString:
if pState.IsDoubleQuotedString {
return errors.New("Escaped single quoted string inside double quoted string...")
} else if pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenSpace:
// Spaces only makes difference inside quotes
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenNewline:
// New lines only makes difference inside quotes
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenSemiColon:
if pState.IsSingleQuotedString || pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
*listCommands = append(*listCommands, command)
command = engine.Command{}
pState = parserState{}
}
case TokenNumbers:
tokenValue := string(t.Bytes())
if pState.IsSingleQuotedString || pState.IsDoubleQuotedString {
setQuotedString(tokenValue, &command, &pState)
} else if pState.IsUsing {
if index.ValidateIndexName(tokenValue) {
command.Index = tokenValue
pState.IsUsing = false
// TokenNumbers is the key of command?
// using document.db mergeset <TokenNumbers> ...
}
} else if pState.IsCommand {
var (
keyBytes []byte
keyType uint8
)
if strings.Contains(tokenValue, ".") {
tokenFloatValue, err := strconv.ParseFloat(tokenValue, 64)
if err != nil {
return fmt.Errorf("Failed to convert %s to float", tokenValue)
}
keyBytes = utils.Float64ToBytes(tokenFloatValue)
keyType = engine.TypeFloat
} else {
tokenIntValue, err := strconv.Atoi(tokenValue)
if err != nil {
return fmt.Errorf("Failed to convert %s to integer", tokenValue)
}
if pState.KVType == engine.TypeUint {
keyBytes = utils.Uint64ToBytes(uint64(tokenIntValue))
keyType = engine.TypeUint
} else if pState.KVType == engine.TypeFloat {
keyBytes = utils.Float64ToBytes(float64(tokenIntValue))
keyType = engine.TypeFloat
} else {
keyBytes = utils.Int64ToBytes(int64(tokenIntValue))
keyType = engine.TypeInt
}
}
command.Key = keyBytes
command.KeyType = keyType
pState.IsCommand = false
pState.IsValue = true
pState.KVType = 0
// TokenNumbers is the command value?
// using document.db mergeset name <TokenNumbers>
} else if pState.IsValue {
var (
valueBytes []byte
valueType uint8
tokenIntValue int64
tokenFloatValue float64
err error
)
if strings.Contains(tokenValue, ".") {
tokenFloatValue, err = strconv.ParseFloat(tokenValue, 64)
if err != nil {
return fmt.Errorf("Failed to convert %s to float", tokenValue)
}
valueBytes = utils.Float64ToBytes(tokenFloatValue)
valueType = engine.TypeFloat
} else {
tokenInt, err := strconv.Atoi(tokenValue)
if err != nil {
return fmt.Errorf("Failed to convert %s to integer", tokenValue)
}
tokenIntValue = int64(tokenInt)
valueBytes = utils.Int64ToBytes(int64(tokenIntValue))
valueType = engine.TypeInt
}
if command.Command == "mergeset" {
if valueType == engine.TypeFloat {
return fmt.Errorf("Failed to parse command. "+
"MergeSet value shall be a unsigned integer "+
"value: %v", tokenValue)
}
command.Value = utils.Uint64ToBytes(uint64(tokenIntValue))
command.ValueType = engine.TypeUint
} else {
command.Value = valueBytes
command.ValueType = valueType
}
pState.IsValue = false
}
default:
return errors.New("Failed to parse line at '" + string(t.Bytes()) + "'")
}
lastTokenType = t.Type()
}
// Checks if the last command was correctly parsed but
// doesn't have the semicolon at the end...
if validateCommand(command) {
*listCommands = append(*listCommands, command)
command = engine.Command{}
pState = parserState{}
// Checks if exists a invalid partial command
} else if command.Index != "" || command.Command != "" ||
command.Key != nil || command.Value != nil {
return fmt.Errorf("The last command wasn't correctly finished nor have the semicolon at end: %v", command)
}
return nil
}