func TestIndexBuildBoolCommands(t *testing.T) {
var (
idx *Index
err error
indexDir = DataDirTmp + "/test-bool"
commands []engine.Command
cmd engine.Command
)
cfg := Config{
Debug: false,
DataDir: DataDirTmp,
}
err = os.MkdirAll(DataDirTmp, 0755)
if err != nil {
t.Error("Failed to create directory")
goto cleanup
}
idx, err = New("test-bool", cfg, true)
if err != nil {
t.Error(err)
goto cleanup
}
commands, err = idx.buildIndexBool(uint64(1), "teste", true)
if err != nil {
t.Error(err)
goto cleanup
}
if len(commands) != 1 {
t.Error("invalid commands returned by buildUint")
goto cleanup
}
cmd = commands[0]
if cmd.Index != "test-bool" ||
cmd.Database != "teste_bool.idx" ||
utils.BytesToBool(cmd.Key) != true ||
utils.BytesToUint64(cmd.Value) != uint64(1) ||
strings.ToLower(cmd.Command) != "mergeset" {
t.Error("commands differs")
fmt.Println("Key: ", utils.BytesToUint64(cmd.Key))
fmt.Println("Value: ", utils.BytesToUint64(cmd.Value))
fmt.Println("Index: ", cmd.Index)
cmd.Println()
goto cleanup
}
cleanup:
idx.Close()
os.RemoveAll(indexDir)
}
func (i *Index) FilterTermID(field, value []byte, limit uint64) ([]uint64, uint64, error) {
cmd := engine.Command{}
cmd.Index = i.Name
// TODO: implement search for every type
cmd.Database = utils.FieldNorm(string(field)) + "_string.idx"
cmd.Command = "get"
cmd.Key = value
cmd.KeyType = engine.TypeString
data, err := i.engine.Execute(cmd)
if err != nil {
return nil, 0, err
}
dataLimit := uint64(len(data) / 8)
total := dataLimit
if limit > 0 && limit < dataLimit {
dataLimit = limit
}
docIDs := make([]uint64, dataLimit)
if len(data) > 0 {
for i, j := uint64(0), uint64(0); i < dataLimit*8; i, j = i+8, j+1 {
v := utils.BytesToUint64(data[i : i+8])
docIDs[j] = v
}
}
return docIDs, total, nil
}
// MergeSet add value to a ordered set of integers stored in key. If value
// is already on the key, than the set will be skipped.
func (lvdb *LVDB) MergeSet(key []byte, value uint64) error {
var (
buf *bytes.Buffer
err error
v uint64
i uint64
inserted bool
)
data, err := lvdb.Get(key)
if err != nil {
return err
}
if lvdb.Config.Debug {
fmt.Printf("[INFO] %d ids == %d GB of ids\n", len(data)/8, len(data)/(1024*1024*1024))
}
buf = new(bytes.Buffer)
lenBytes := uint64(len(data))
// O(n)
for i = 0; i < lenBytes; i += 8 {
v = utils.BytesToUint64(data[i : i+8])
// returns if value is already stored
if v == value {
return nil
}
if value < v {
err = binary.Write(buf, binary.BigEndian, value)
if err != nil {
return err
}
inserted = true
}
err = binary.Write(buf, binary.BigEndian, v)
if err != nil {
return err
}
}
if lenBytes == 0 || !inserted {
err = binary.Write(buf, binary.BigEndian, value)
if err != nil {
return err
}
}
return lvdb.Set(key, buf.Bytes())
}
func (i *Index) matchPrefix(field []byte, value []byte) ([]uint64, error) {
var (
docIDs []uint64
)
// TODO: Implement search for all of field types
storekv, err := i.engine.GetStore(i.Name, utils.FieldNorm(string(field))+"_string.idx")
if err != nil {
return nil, err
}
it := storekv.GetIterator()
defer it.Close()
for it.Seek(value); it.Valid(); it.Next() {
if bytes.HasPrefix(it.Key(), value) {
var ids []uint64
dataBytes := it.Value()
if len(dataBytes) == 0 {
continue
}
for i := 0; i < len(dataBytes); i += 8 {
v := utils.BytesToUint64(dataBytes[i : i+8])
ids = append(ids, v)
}
if len(docIDs) == 0 {
docIDs = ids
continue
}
for _, id := range ids {
docIDs = utils.UniqueUint64Add(docIDs, id)
}
}
}
if err := it.GetError(); err != nil {
return nil, err
}
return docIDs, nil
}