func TestIndexBuildBoolCommands(t *testing.T) {
var (
idx *Index
err error
indexDir = DataDirTmp + "/test-bool"
commands []engine.Command
cmd engine.Command
)
cfg := Config{
Debug: false,
DataDir: DataDirTmp,
}
err = os.MkdirAll(DataDirTmp, 0755)
if err != nil {
t.Error("Failed to create directory")
goto cleanup
}
idx, err = New("test-bool", cfg, true)
if err != nil {
t.Error(err)
goto cleanup
}
commands, err = idx.buildIndexBool(uint64(1), "teste", true)
if err != nil {
t.Error(err)
goto cleanup
}
if len(commands) != 1 {
t.Error("invalid commands returned by buildUint")
goto cleanup
}
cmd = commands[0]
if cmd.Index != "test-bool" ||
cmd.Database != "teste_bool.idx" ||
utils.BytesToBool(cmd.Key) != true ||
utils.BytesToUint64(cmd.Value) != uint64(1) ||
strings.ToLower(cmd.Command) != "mergeset" {
t.Error("commands differs")
fmt.Println("Key: ", utils.BytesToUint64(cmd.Key))
fmt.Println("Value: ", utils.BytesToUint64(cmd.Value))
fmt.Println("Index: ", cmd.Index)
cmd.Println()
goto cleanup
}
cleanup:
idx.Close()
os.RemoveAll(indexDir)
}
func setQuotedString(token string, command *engine.Command, pState *parserState) {
if pState.IsUsing {
command.Index += token
} else if pState.IsCommand {
command.Key = []byte(string(command.Key) + token)
command.KeyType = engine.TypeString
} else if pState.IsValue {
command.Value = []byte(string(command.Value) + token)
command.ValueType = engine.TypeString
}
}
func (handler *GetAnalyseHandler) ServeHTTP(res http.ResponseWriter, req *http.Request, ps httprouter.Params) {
var (
err error
cmd engine.Command
exists bool
)
handler.ProcessVars(ps)
indexName := handler.GetIndexName()
if exists, err = handler.search.IndexExists(indexName); exists != true && err == nil {
response := map[string]string{
"error": "Index '" + indexName + "' doesn't exists.",
}
handler.WriteJSONObject(res, response)
return
} else if exists == false && err != nil {
res.WriteHeader(http.StatusInternalServerError)
handler.Error(res, err.Error())
return
}
docID := handler.GetDocumentID()
docIntID, err := strconv.Atoi(docID)
if err != nil {
res.WriteHeader(http.StatusBadRequest)
handler.Error(res, "Invalid document id: "+docID)
return
}
index, err := handler.search.OpenIndex(indexName)
if err != nil {
handler.Error(res, err.Error())
return
}
cmd, err = index.GetAnalyze(uint64(docIntID))
if err != nil {
res.WriteHeader(http.StatusBadRequest)
handler.Error(res, err.Error())
return
}
res.Write([]byte(cmd.Reverse()))
}
func (i *Index) FilterTermID(field, value []byte, limit uint64) ([]uint64, uint64, error) {
cmd := engine.Command{}
cmd.Index = i.Name
// TODO: implement search for every type
cmd.Database = utils.FieldNorm(string(field)) + "_string.idx"
cmd.Command = "get"
cmd.Key = value
cmd.KeyType = engine.TypeString
data, err := i.engine.Execute(cmd)
if err != nil {
return nil, 0, err
}
dataLimit := uint64(len(data) / 8)
total := dataLimit
if limit > 0 && limit < dataLimit {
dataLimit = limit
}
docIDs := make([]uint64, dataLimit)
if len(data) > 0 {
for i, j := uint64(0), uint64(0); i < dataLimit*8; i, j = i+8, j+1 {
v := utils.BytesToUint64(data[i : i+8])
docIDs[j] = v
}
}
return docIDs, total, nil
}
func compareCommand(cmd engine.Command, expected engine.Command, t *testing.T) {
if !reflect.DeepEqual(cmd, expected) {
t.Errorf("Unexpected parsed command: %v !== %v", cmd.Reverse(), expected.Reverse())
fmt.Printf("%v !== %v\n", cmd, expected)
}
}
// FromReader parse the file
func FromReader(file io.Reader, listCommands *[]engine.Command) error {
var command engine.Command
pState := parserState{}
// Create our lexer
// NewSize(startState, reader, readerBufLen, channelCap)
lex := lexer.NewSize(lexFunc, file, 100, 1)
var lastTokenType = TokenNil
// Process lexer-emitted tokens
for t := lex.NextToken(); lexer.TokenTypeEOF != t.Type(); t = lex.NextToken() {
switch t.Type() {
case TokenWord:
if lastTokenType != TokenWord {
tokenValue := string(t.Bytes())
// TokenWord is a double or single quoted string?
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
if !pState.IsUsing && !pState.IsCommand && !pState.IsValue {
return errors.New("Invalid quoted string: " + tokenValue)
}
setQuotedString(tokenValue, &command, &pState)
// TokenWord is the Index name?
// using <TokenWord> ...
} else if pState.IsUsing {
indexDbParts := strings.Split(tokenValue, ".")
if len(indexDbParts) < 2 {
return fmt.Errorf("Invalid USING <index>.<database>: %s", tokenValue)
}
command.Index = indexDbParts[0]
command.Database = strings.Join(indexDbParts[1:], ".")
pState.IsUsing = false
// TokenWord is the key of command?
// using document.db mergeset <TokenWord> ...
} else if pState.IsCommand {
if strings.HasPrefix(tokenValue, "int(") {
pState.KVType = engine.TypeInt
pState.IsCastOpen = true
} else if strings.HasPrefix(tokenValue, "uint(") {
pState.KVType = engine.TypeUint
pState.IsCastOpen = true
} else if strings.HasPrefix(tokenValue, "float(") {
pState.KVType = engine.TypeFloat
pState.IsCastOpen = true
} else {
command.Key = []byte(tokenValue)
command.KeyType = engine.TypeString
pState.IsCommand = false
pState.IsValue = true
}
// TokenWord is the command value?
// using document.db mergeset name <TokenWord>
} else if pState.IsValue {
if tokenValue == ")" && pState.IsCastOpen {
pState.IsCastOpen = false
} else if strings.HasPrefix(tokenValue, "uint(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeUint
} else if strings.HasPrefix(tokenValue, "int(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeInt
} else if strings.HasPrefix(tokenValue, "float(") {
pState.IsCastOpen = true
pState.KVType = engine.TypeFloat
} else {
command.Value = []byte(tokenValue)
command.ValueType = engine.TypeString
pState.IsValue = false
}
} else {
// Here we handle the available KEYWORDS
if pState.IsCastOpen && strings.HasPrefix(tokenValue, ")") {
pState.IsCastOpen = false
pState.KVType = 0
// Keyword USING
} else if !pState.IsUsing && strings.ToLower(tokenValue) == "using" {
pState.IsUsing = true
} else if !pState.IsCommand {
// Must be a command KEYWORD
// see commandsAvailable
if !isValidCommand(tokenValue) {
return fmt.Errorf("Invalid keyword '"+tokenValue+"': %s", command)
}
pState.IsCommand = true
pState.IsUsing = false
command.Command = strings.ToLower(tokenValue)
}
}
}
case TokenDoubleQuotedString:
if pState.IsDoubleQuotedString {
if pState.IsCommand {
pState.IsCommand = false
pState.IsValue = true
} else if pState.IsUsing {
pState.IsUsing = false
} else if pState.IsValue {
pState.IsValue = false
}
}
if pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
pState.IsDoubleQuotedString = !pState.IsDoubleQuotedString
}
case TokenSingleQuotedString:
if pState.IsSingleQuotedString {
if pState.IsCommand {
pState.IsCommand = false
pState.IsValue = true
} else if pState.IsUsing {
pState.IsUsing = false
} else if pState.IsValue {
pState.IsValue = false
}
}
if pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
pState.IsSingleQuotedString = !pState.IsSingleQuotedString
}
case TokenEscapedDoubleQuotedString:
if pState.IsSingleQuotedString {
panic("Escaped double quoted string inside single quoted string...")
} else if pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()[1:]), &command, &pState)
}
pState.IsEscapedDoubleQuotedString = !pState.IsEscapedDoubleQuotedString
case TokenEscapedSingleQuotedString:
if pState.IsDoubleQuotedString {
return errors.New("Escaped single quoted string inside double quoted string...")
} else if pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenSpace:
// Spaces only makes difference inside quotes
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenNewline:
// New lines only makes difference inside quotes
if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
}
case TokenSemiColon:
if pState.IsSingleQuotedString || pState.IsDoubleQuotedString {
setQuotedString(string(t.Bytes()), &command, &pState)
} else {
*listCommands = append(*listCommands, command)
command = engine.Command{}
pState = parserState{}
}
case TokenNumbers:
tokenValue := string(t.Bytes())
if pState.IsSingleQuotedString || pState.IsDoubleQuotedString {
setQuotedString(tokenValue, &command, &pState)
} else if pState.IsUsing {
if index.ValidateIndexName(tokenValue) {
command.Index = tokenValue
pState.IsUsing = false
// TokenNumbers is the key of command?
// using document.db mergeset <TokenNumbers> ...
}
} else if pState.IsCommand {
var (
keyBytes []byte
keyType uint8
)
if strings.Contains(tokenValue, ".") {
tokenFloatValue, err := strconv.ParseFloat(tokenValue, 64)
if err != nil {
return fmt.Errorf("Failed to convert %s to float", tokenValue)
}
keyBytes = utils.Float64ToBytes(tokenFloatValue)
keyType = engine.TypeFloat
} else {
tokenIntValue, err := strconv.Atoi(tokenValue)
if err != nil {
return fmt.Errorf("Failed to convert %s to integer", tokenValue)
}
if pState.KVType == engine.TypeUint {
keyBytes = utils.Uint64ToBytes(uint64(tokenIntValue))
keyType = engine.TypeUint
} else if pState.KVType == engine.TypeFloat {
keyBytes = utils.Float64ToBytes(float64(tokenIntValue))
keyType = engine.TypeFloat
} else {
keyBytes = utils.Int64ToBytes(int64(tokenIntValue))
keyType = engine.TypeInt
}
}
command.Key = keyBytes
command.KeyType = keyType
pState.IsCommand = false
pState.IsValue = true
pState.KVType = 0
// TokenNumbers is the command value?
// using document.db mergeset name <TokenNumbers>
} else if pState.IsValue {
var (
valueBytes []byte
valueType uint8
tokenIntValue int64
tokenFloatValue float64
err error
)
if strings.Contains(tokenValue, ".") {
tokenFloatValue, err = strconv.ParseFloat(tokenValue, 64)
if err != nil {
return fmt.Errorf("Failed to convert %s to float", tokenValue)
}
valueBytes = utils.Float64ToBytes(tokenFloatValue)
valueType = engine.TypeFloat
} else {
tokenInt, err := strconv.Atoi(tokenValue)
if err != nil {
return fmt.Errorf("Failed to convert %s to integer", tokenValue)
}
tokenIntValue = int64(tokenInt)
valueBytes = utils.Int64ToBytes(int64(tokenIntValue))
valueType = engine.TypeInt
}
if command.Command == "mergeset" {
if valueType == engine.TypeFloat {
return fmt.Errorf("Failed to parse command. "+
"MergeSet value shall be a unsigned integer "+
"value: %v", tokenValue)
}
command.Value = utils.Uint64ToBytes(uint64(tokenIntValue))
command.ValueType = engine.TypeUint
} else {
command.Value = valueBytes
command.ValueType = valueType
}
pState.IsValue = false
}
default:
return errors.New("Failed to parse line at '" + string(t.Bytes()) + "'")
}
lastTokenType = t.Type()
}
// Checks if the last command was correctly parsed but
// doesn't have the semicolon at the end...
if validateCommand(command) {
*listCommands = append(*listCommands, command)
command = engine.Command{}
pState = parserState{}
// Checks if exists a invalid partial command
} else if command.Index != "" || command.Command != "" ||
command.Key != nil || command.Value != nil {
return fmt.Errorf("The last command wasn't correctly finished nor have the semicolon at end: %v", command)
}
return nil
}