Ejemplo n.º 1
0
func TestIndexBuildBoolCommands(t *testing.T) {
	var (
		idx      *Index
		err      error
		indexDir = DataDirTmp + "/test-bool"
		commands []engine.Command
		cmd      engine.Command
	)

	cfg := Config{
		Debug:   false,
		DataDir: DataDirTmp,
	}

	err = os.MkdirAll(DataDirTmp, 0755)

	if err != nil {
		t.Error("Failed to create directory")
		goto cleanup
	}

	idx, err = New("test-bool", cfg, true)

	if err != nil {
		t.Error(err)
		goto cleanup
	}

	commands, err = idx.buildIndexBool(uint64(1), "teste", true)

	if err != nil {
		t.Error(err)
		goto cleanup
	}

	if len(commands) != 1 {
		t.Error("invalid commands returned by buildUint")
		goto cleanup
	}

	cmd = commands[0]

	if cmd.Index != "test-bool" ||
		cmd.Database != "teste_bool.idx" ||
		utils.BytesToBool(cmd.Key) != true ||
		utils.BytesToUint64(cmd.Value) != uint64(1) ||
		strings.ToLower(cmd.Command) != "mergeset" {
		t.Error("commands differs")
		fmt.Println("Key: ", utils.BytesToUint64(cmd.Key))
		fmt.Println("Value: ", utils.BytesToUint64(cmd.Value))
		fmt.Println("Index: ", cmd.Index)
		cmd.Println()
		goto cleanup
	}

cleanup:
	idx.Close()
	os.RemoveAll(indexDir)
}
Ejemplo n.º 2
0
func setQuotedString(token string, command *engine.Command, pState *parserState) {
	if pState.IsUsing {
		command.Index += token
	} else if pState.IsCommand {
		command.Key = []byte(string(command.Key) + token)
		command.KeyType = engine.TypeString
	} else if pState.IsValue {
		command.Value = []byte(string(command.Value) + token)
		command.ValueType = engine.TypeString
	}
}
Ejemplo n.º 3
0
func (handler *GetAnalyseHandler) ServeHTTP(res http.ResponseWriter, req *http.Request, ps httprouter.Params) {
	var (
		err    error
		cmd    engine.Command
		exists bool
	)

	handler.ProcessVars(ps)
	indexName := handler.GetIndexName()

	if exists, err = handler.search.IndexExists(indexName); exists != true && err == nil {
		response := map[string]string{
			"error": "Index '" + indexName + "' doesn't exists.",
		}

		handler.WriteJSONObject(res, response)
		return
	} else if exists == false && err != nil {
		res.WriteHeader(http.StatusInternalServerError)
		handler.Error(res, err.Error())
		return
	}

	docID := handler.GetDocumentID()

	docIntID, err := strconv.Atoi(docID)

	if err != nil {
		res.WriteHeader(http.StatusBadRequest)
		handler.Error(res, "Invalid document id: "+docID)
		return
	}

	index, err := handler.search.OpenIndex(indexName)

	if err != nil {
		handler.Error(res, err.Error())
		return
	}

	cmd, err = index.GetAnalyze(uint64(docIntID))

	if err != nil {
		res.WriteHeader(http.StatusBadRequest)
		handler.Error(res, err.Error())
		return
	}

	res.Write([]byte(cmd.Reverse()))
}
Ejemplo n.º 4
0
func (i *Index) FilterTermID(field, value []byte, limit uint64) ([]uint64, uint64, error) {
	cmd := engine.Command{}
	cmd.Index = i.Name
	// TODO: implement search for every type
	cmd.Database = utils.FieldNorm(string(field)) + "_string.idx"
	cmd.Command = "get"
	cmd.Key = value
	cmd.KeyType = engine.TypeString
	data, err := i.engine.Execute(cmd)

	if err != nil {
		return nil, 0, err
	}

	dataLimit := uint64(len(data) / 8)
	total := dataLimit

	if limit > 0 && limit < dataLimit {
		dataLimit = limit
	}

	docIDs := make([]uint64, dataLimit)

	if len(data) > 0 {
		for i, j := uint64(0), uint64(0); i < dataLimit*8; i, j = i+8, j+1 {
			v := utils.BytesToUint64(data[i : i+8])
			docIDs[j] = v
		}

	}

	return docIDs, total, nil
}
Ejemplo n.º 5
0
func compareCommand(cmd engine.Command, expected engine.Command, t *testing.T) {
	if !reflect.DeepEqual(cmd, expected) {
		t.Errorf("Unexpected parsed command: %v !== %v", cmd.Reverse(), expected.Reverse())
		fmt.Printf("%v !== %v\n", cmd, expected)
	}
}
Ejemplo n.º 6
0
// FromReader parse the file
func FromReader(file io.Reader, listCommands *[]engine.Command) error {
	var command engine.Command

	pState := parserState{}

	// Create our lexer
	// NewSize(startState, reader, readerBufLen, channelCap)
	lex := lexer.NewSize(lexFunc, file, 100, 1)
	var lastTokenType = TokenNil

	// Process lexer-emitted tokens
	for t := lex.NextToken(); lexer.TokenTypeEOF != t.Type(); t = lex.NextToken() {
		switch t.Type() {
		case TokenWord:
			if lastTokenType != TokenWord {
				tokenValue := string(t.Bytes())

				// TokenWord is a double or single quoted string?
				if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
					if !pState.IsUsing && !pState.IsCommand && !pState.IsValue {
						return errors.New("Invalid quoted string: " + tokenValue)
					}

					setQuotedString(tokenValue, &command, &pState)

					// TokenWord is the Index name?
					// using <TokenWord> ...
				} else if pState.IsUsing {
					indexDbParts := strings.Split(tokenValue, ".")

					if len(indexDbParts) < 2 {
						return fmt.Errorf("Invalid USING <index>.<database>: %s", tokenValue)
					}

					command.Index = indexDbParts[0]
					command.Database = strings.Join(indexDbParts[1:], ".")
					pState.IsUsing = false

					// TokenWord is the key of command?
					// using document.db mergeset <TokenWord> ...
				} else if pState.IsCommand {
					if strings.HasPrefix(tokenValue, "int(") {
						pState.KVType = engine.TypeInt
						pState.IsCastOpen = true
					} else if strings.HasPrefix(tokenValue, "uint(") {
						pState.KVType = engine.TypeUint
						pState.IsCastOpen = true
					} else if strings.HasPrefix(tokenValue, "float(") {
						pState.KVType = engine.TypeFloat
						pState.IsCastOpen = true
					} else {
						command.Key = []byte(tokenValue)
						command.KeyType = engine.TypeString
						pState.IsCommand = false
						pState.IsValue = true
					}

					// TokenWord is the command value?
					// using document.db mergeset name <TokenWord>
				} else if pState.IsValue {
					if tokenValue == ")" && pState.IsCastOpen {
						pState.IsCastOpen = false
					} else if strings.HasPrefix(tokenValue, "uint(") {
						pState.IsCastOpen = true
						pState.KVType = engine.TypeUint
					} else if strings.HasPrefix(tokenValue, "int(") {
						pState.IsCastOpen = true
						pState.KVType = engine.TypeInt
					} else if strings.HasPrefix(tokenValue, "float(") {
						pState.IsCastOpen = true
						pState.KVType = engine.TypeFloat
					} else {
						command.Value = []byte(tokenValue)
						command.ValueType = engine.TypeString
						pState.IsValue = false
					}
				} else {
					// Here we handle the available KEYWORDS

					if pState.IsCastOpen && strings.HasPrefix(tokenValue, ")") {
						pState.IsCastOpen = false
						pState.KVType = 0

						// Keyword USING
					} else if !pState.IsUsing && strings.ToLower(tokenValue) == "using" {
						pState.IsUsing = true
					} else if !pState.IsCommand {
						// Must be a command KEYWORD
						// see commandsAvailable

						if !isValidCommand(tokenValue) {
							return fmt.Errorf("Invalid keyword '"+tokenValue+"': %s", command)
						}

						pState.IsCommand = true
						pState.IsUsing = false
						command.Command = strings.ToLower(tokenValue)
					}
				}
			}
		case TokenDoubleQuotedString:
			if pState.IsDoubleQuotedString {
				if pState.IsCommand {
					pState.IsCommand = false
					pState.IsValue = true
				} else if pState.IsUsing {
					pState.IsUsing = false
				} else if pState.IsValue {
					pState.IsValue = false
				}
			}

			if pState.IsSingleQuotedString {
				setQuotedString(string(t.Bytes()), &command, &pState)
			} else {
				pState.IsDoubleQuotedString = !pState.IsDoubleQuotedString
			}
		case TokenSingleQuotedString:
			if pState.IsSingleQuotedString {
				if pState.IsCommand {
					pState.IsCommand = false
					pState.IsValue = true
				} else if pState.IsUsing {
					pState.IsUsing = false
				} else if pState.IsValue {
					pState.IsValue = false
				}
			}

			if pState.IsDoubleQuotedString {
				setQuotedString(string(t.Bytes()), &command, &pState)
			} else {
				pState.IsSingleQuotedString = !pState.IsSingleQuotedString
			}

		case TokenEscapedDoubleQuotedString:
			if pState.IsSingleQuotedString {
				panic("Escaped double quoted string inside single quoted string...")
			} else if pState.IsDoubleQuotedString {
				setQuotedString(string(t.Bytes()[1:]), &command, &pState)
			}

			pState.IsEscapedDoubleQuotedString = !pState.IsEscapedDoubleQuotedString

		case TokenEscapedSingleQuotedString:
			if pState.IsDoubleQuotedString {
				return errors.New("Escaped single quoted string inside double quoted string...")
			} else if pState.IsSingleQuotedString {
				setQuotedString(string(t.Bytes()), &command, &pState)
			}
		case TokenSpace:
			// Spaces only makes difference inside quotes
			if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
				setQuotedString(string(t.Bytes()), &command, &pState)
			}

		case TokenNewline:
			// New lines only makes difference inside quotes
			if pState.IsDoubleQuotedString || pState.IsSingleQuotedString {
				setQuotedString(string(t.Bytes()), &command, &pState)
			}
		case TokenSemiColon:
			if pState.IsSingleQuotedString || pState.IsDoubleQuotedString {
				setQuotedString(string(t.Bytes()), &command, &pState)
			} else {
				*listCommands = append(*listCommands, command)
				command = engine.Command{}
				pState = parserState{}
			}
		case TokenNumbers:
			tokenValue := string(t.Bytes())

			if pState.IsSingleQuotedString || pState.IsDoubleQuotedString {
				setQuotedString(tokenValue, &command, &pState)
			} else if pState.IsUsing {
				if index.ValidateIndexName(tokenValue) {
					command.Index = tokenValue
					pState.IsUsing = false

					// TokenNumbers is the key of command?
					// using document.db mergeset <TokenNumbers> ...
				}
			} else if pState.IsCommand {
				var (
					keyBytes []byte
					keyType  uint8
				)

				if strings.Contains(tokenValue, ".") {
					tokenFloatValue, err := strconv.ParseFloat(tokenValue, 64)

					if err != nil {
						return fmt.Errorf("Failed to convert %s to float", tokenValue)
					}

					keyBytes = utils.Float64ToBytes(tokenFloatValue)
					keyType = engine.TypeFloat
				} else {
					tokenIntValue, err := strconv.Atoi(tokenValue)

					if err != nil {
						return fmt.Errorf("Failed to convert %s to integer", tokenValue)
					}

					if pState.KVType == engine.TypeUint {
						keyBytes = utils.Uint64ToBytes(uint64(tokenIntValue))
						keyType = engine.TypeUint
					} else if pState.KVType == engine.TypeFloat {
						keyBytes = utils.Float64ToBytes(float64(tokenIntValue))
						keyType = engine.TypeFloat
					} else {
						keyBytes = utils.Int64ToBytes(int64(tokenIntValue))
						keyType = engine.TypeInt
					}
				}

				command.Key = keyBytes
				command.KeyType = keyType
				pState.IsCommand = false
				pState.IsValue = true

				pState.KVType = 0

				// TokenNumbers is the command value?
				// using document.db mergeset name <TokenNumbers>
			} else if pState.IsValue {
				var (
					valueBytes      []byte
					valueType       uint8
					tokenIntValue   int64
					tokenFloatValue float64
					err             error
				)

				if strings.Contains(tokenValue, ".") {
					tokenFloatValue, err = strconv.ParseFloat(tokenValue, 64)

					if err != nil {
						return fmt.Errorf("Failed to convert %s to float", tokenValue)
					}

					valueBytes = utils.Float64ToBytes(tokenFloatValue)
					valueType = engine.TypeFloat
				} else {
					tokenInt, err := strconv.Atoi(tokenValue)

					if err != nil {
						return fmt.Errorf("Failed to convert %s to integer", tokenValue)
					}

					tokenIntValue = int64(tokenInt)
					valueBytes = utils.Int64ToBytes(int64(tokenIntValue))
					valueType = engine.TypeInt
				}

				if command.Command == "mergeset" {
					if valueType == engine.TypeFloat {
						return fmt.Errorf("Failed to parse command. "+
							"MergeSet value shall be a unsigned integer "+
							"value: %v", tokenValue)
					}

					command.Value = utils.Uint64ToBytes(uint64(tokenIntValue))
					command.ValueType = engine.TypeUint
				} else {
					command.Value = valueBytes
					command.ValueType = valueType
				}

				pState.IsValue = false
			}
		default:
			return errors.New("Failed to parse line at '" + string(t.Bytes()) + "'")
		}

		lastTokenType = t.Type()
	}

	// Checks if the last command was correctly parsed but
	// doesn't have the semicolon at the end...
	if validateCommand(command) {
		*listCommands = append(*listCommands, command)
		command = engine.Command{}
		pState = parserState{}

		// Checks if exists a invalid partial command
	} else if command.Index != "" || command.Command != "" ||
		command.Key != nil || command.Value != nil {
		return fmt.Errorf("The last command wasn't correctly finished nor have the semicolon at end: %v", command)
	}

	return nil
}