func CommonTestMerge(t *testing.T, s store.KVStore) { testKey := []byte("k1") data := []struct { key []byte val []byte }{ {testKey, encodeUint64(1)}, {testKey, encodeUint64(1)}, } // open a writer writer, err := s.Writer() if err != nil { t.Fatal(err) } // write the data batch := writer.NewBatch() for _, row := range data { batch.Merge(row.key, row.val) } err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } // close the writer err = writer.Close() if err != nil { t.Fatal(err) } // open a reader reader, err := s.Reader() if err != nil { t.Fatal(err) } // read key returnedVal, err := reader.Get(testKey) if err != nil { t.Fatal(err) } // check the value mergedval := binary.LittleEndian.Uint64(returnedVal) if mergedval != 2 { t.Errorf("expected 2, got %d", mergedval) } // close the reader err = reader.Close() if err != nil { t.Fatal(err) } }
func cleanup(t *testing.T, s store.KVStore) { err := s.Close() if err != nil { t.Fatal(err) } err = os.RemoveAll("test") if err != nil { t.Fatal(err) } }
// NullTestKVStore has very different expectations // compared to CommonTestKVStore func NullTestKVStore(t *testing.T, s store.KVStore) { writer, err := s.Writer() if err != nil { t.Error(err) } batch := writer.NewBatch() batch.Set([]byte("b"), []byte("val-b")) batch.Set([]byte("c"), []byte("val-c")) batch.Set([]byte("d"), []byte("val-d")) batch.Set([]byte("e"), []byte("val-e")) batch.Set([]byte("f"), []byte("val-f")) batch.Set([]byte("g"), []byte("val-g")) batch.Set([]byte("h"), []byte("val-h")) batch.Set([]byte("i"), []byte("val-i")) batch.Set([]byte("j"), []byte("val-j")) err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } err = writer.Close() if err != nil { t.Fatal(err) } reader, err := s.Reader() if err != nil { t.Error(err) } defer func() { err := reader.Close() if err != nil { t.Fatal(err) } }() it := reader.RangeIterator([]byte("b"), nil) key, val, valid := it.Current() if valid { t.Fatalf("valid true, expected false") } if key != nil { t.Fatalf("expected key nil, got %s", key) } if val != nil { t.Fatalf("expected value nil, got %s", val) } err = it.Close() if err != nil { t.Fatal(err) } err = s.Close() if err != nil { t.Fatal(err) } }
func batchWriteRows(s store.KVStore, rows []testRow) error { // open a writer writer, err := s.Writer() if err != nil { return err } // write the data batch := writer.NewBatch() for _, row := range rows { batch.Set(row.key, row.val) } err = writer.ExecuteBatch(batch) if err != nil { return err } // close the writer err = writer.Close() if err != nil { return err } return nil }
func cleanup(t *testing.T, s store.KVStore) { err := s.Close() if err != nil { t.Fatal(err) } }
func CommonTestKVCrud(t *testing.T, s store.KVStore) { writer, err := s.Writer() if err != nil { t.Error(err) } batch := writer.NewBatch() batch.Set([]byte("a"), []byte("val-a")) batch.Set([]byte("z"), []byte("val-z")) batch.Delete([]byte("z")) err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } batch.Reset() batch.Set([]byte("b"), []byte("val-b")) batch.Set([]byte("c"), []byte("val-c")) batch.Set([]byte("d"), []byte("val-d")) batch.Set([]byte("e"), []byte("val-e")) batch.Set([]byte("f"), []byte("val-f")) batch.Set([]byte("g"), []byte("val-g")) batch.Set([]byte("h"), []byte("val-h")) batch.Set([]byte("i"), []byte("val-i")) batch.Set([]byte("j"), []byte("val-j")) err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } err = writer.Close() if err != nil { t.Fatal(err) } reader, err := s.Reader() if err != nil { t.Error(err) } defer func() { err := reader.Close() if err != nil { t.Fatal(err) } }() it := reader.RangeIterator([]byte("b"), nil) key, val, valid := it.Current() if !valid { t.Fatalf("valid false, expected true") } if string(key) != "b" { t.Fatalf("expected key b, got %s", key) } if string(val) != "val-b" { t.Fatalf("expected value val-b, got %s", val) } it.Next() key, val, valid = it.Current() if !valid { t.Fatalf("valid false, expected true") } if string(key) != "c" { t.Fatalf("expected key c, got %s", key) } if string(val) != "val-c" { t.Fatalf("expected value val-c, got %s", val) } it.Seek([]byte("i")) key, val, valid = it.Current() if !valid { t.Fatalf("valid false, expected true") } if string(key) != "i" { t.Fatalf("expected key i, got %s", key) } if string(val) != "val-i" { t.Fatalf("expected value val-i, got %s", val) } err = it.Close() if err != nil { t.Fatal(err) } }
// CommonTestReaderOwnsGetBytes attempts to mutate the returned bytes // first, while the reader is still open, second after that reader is // closed, then the original key is read again, to ensure these // modifications did not cause panic, or mutate the stored value func CommonTestReaderOwnsGetBytes(t *testing.T, s store.KVStore) { originalKey := []byte("key") originalVal := []byte("val") // open a writer writer, err := s.Writer() if err != nil { t.Fatal(err) } // write key/val batch := writer.NewBatch() batch.Set(originalKey, originalVal) err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } // close the writer err = writer.Close() if err != nil { t.Fatal(err) } // open a reader reader, err := s.Reader() if err != nil { t.Fatal(err) } // read key returnedVal, err := reader.Get(originalKey) if err != nil { t.Fatal(err) } // check that it is the expected value if !reflect.DeepEqual(returnedVal, originalVal) { t.Fatalf("expected value: %v for '%s', got %v", originalVal, originalKey, returnedVal) } // mutate the returned value with reader still open for i := range returnedVal { returnedVal[i] = '1' } // read the key again returnedVal2, err := reader.Get(originalKey) if err != nil { t.Fatal(err) } // check that it is the expected value if !reflect.DeepEqual(returnedVal2, originalVal) { t.Fatalf("expected value: %v for '%s', got %v", originalVal, originalKey, returnedVal2) } // close the reader err = reader.Close() if err != nil { t.Fatal(err) } // mutate the original returned value again for i := range returnedVal { returnedVal[i] = '2' } // open another reader reader, err = s.Reader() if err != nil { t.Fatal(err) } // read the key again returnedVal3, err := reader.Get(originalKey) if err != nil { t.Fatal(err) } // check that it is the expected value if !reflect.DeepEqual(returnedVal3, originalVal) { t.Fatalf("expected value: %v for '%s', got %v", originalVal, originalKey, returnedVal3) } // close the reader err = reader.Close() if err != nil { t.Fatal(err) } // finally check that the value we mutated still has what we set it to for i := range returnedVal { if returnedVal[i] != '2' { t.Errorf("expected byte to be '2', got %v", returnedVal[i]) } } }
func CommonTestWriterOwnsBytes(t *testing.T, s store.KVStore) { keyBuffer := make([]byte, 5) valBuffer := make([]byte, 5) // open a writer writer, err := s.Writer() if err != nil { t.Fatal(err) } // write key/val pairs reusing same buffer batch := writer.NewBatch() for i := 0; i < 10; i++ { keyBuffer[0] = 'k' keyBuffer[1] = 'e' keyBuffer[2] = 'y' keyBuffer[3] = '-' keyBuffer[4] = byte('0' + i) valBuffer[0] = 'v' valBuffer[1] = 'a' valBuffer[2] = 'l' valBuffer[3] = '-' valBuffer[4] = byte('0' + i) batch.Set(keyBuffer, valBuffer) } err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } // close the writer err = writer.Close() if err != nil { t.Fatal(err) } // open a reader reader, err := s.Reader() if err != nil { t.Fatal(err) } // check that we can read back what we expect allks := make([][]byte, 0) allvs := make([][]byte, 0) iter := reader.RangeIterator(nil, nil) for iter.Valid() { // if we want to keep bytes from iteration we must copy k := iter.Key() copyk := make([]byte, len(k)) copy(copyk, k) allks = append(allks, copyk) v := iter.Key() copyv := make([]byte, len(v)) copy(copyv, v) allvs = append(allvs, copyv) iter.Next() } err = iter.Close() if err != nil { t.Fatal(err) } if len(allks) != 10 { t.Fatalf("expected 10 k/v pairs, got %d", len(allks)) } for i, key := range allks { val := allvs[i] if !bytes.HasSuffix(key, []byte{byte('0' + i)}) { t.Errorf("expected key %v to end in %d", key, []byte{byte('0' + i)}) } if !bytes.HasSuffix(val, []byte{byte('0' + i)}) { t.Errorf("expected val %v to end in %d", val, []byte{byte('0' + i)}) } } // close the reader err = reader.Close() if err != nil { t.Fatal(err) } // open a writer writer, err = s.Writer() if err != nil { t.Fatal(err) } // now delete using same approach batch = writer.NewBatch() for i := 0; i < 10; i++ { keyBuffer[0] = 'k' keyBuffer[1] = 'e' keyBuffer[2] = 'y' keyBuffer[3] = '-' keyBuffer[4] = byte('0' + i) batch.Delete(keyBuffer) } err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } // close the writer err = writer.Close() if err != nil { t.Fatal(err) } // open a reader reader, err = s.Reader() if err != nil { t.Fatal(err) } // check that we can read back what we expect allks = make([][]byte, 0) iter = reader.RangeIterator(nil, nil) for iter.Valid() { // if we want to keep bytes from iteration we must copy k := iter.Key() copyk := make([]byte, len(k)) copy(copyk, k) allks = append(allks, copyk) v := iter.Key() copyv := make([]byte, len(v)) copy(copyv, v) allvs = append(allvs, copyv) iter.Next() } err = iter.Close() if err != nil { t.Fatal(err) } if len(allks) != 0 { t.Fatalf("expected 0 k/v pairs remaining, got %d", len(allks)) } // close the reader err = reader.Close() if err != nil { t.Fatal(err) } }
func CommonTestPrefixIterator(t *testing.T, s store.KVStore) { data := []testRow{ {[]byte("apple"), []byte("val")}, {[]byte("cat1"), []byte("val")}, {[]byte("cat2"), []byte("val")}, {[]byte("cat3"), []byte("val")}, {[]byte("dog1"), []byte("val")}, {[]byte("dog2"), []byte("val")}, {[]byte("dog4"), []byte("val")}, {[]byte("elephant"), []byte("val")}, } expectedCats := [][]byte{ []byte("cat1"), []byte("cat2"), []byte("cat3"), } expectedDogs := [][]byte{ []byte("dog1"), // we seek to "dog3" and ensure it skips over "dog2" // but still finds "dog4" even though there was no "dog3" []byte("dog4"), } err := batchWriteRows(s, data) if err != nil { t.Fatal(err) } // open a reader reader, err := s.Reader() if err != nil { t.Fatal(err) } // get a prefix reader cats := make([][]byte, 0) iter := reader.PrefixIterator([]byte("cat")) for iter.Valid() { // if we want to keep bytes from iteration we must copy k := iter.Key() copyk := make([]byte, len(k)) copy(copyk, k) cats = append(cats, copyk) iter.Next() } err = iter.Close() if err != nil { t.Fatal(err) } // check that we found all the cats if !reflect.DeepEqual(cats, expectedCats) { t.Fatalf("expected cats %v, got %v", expectedCats, cats) } // get a prefix reader dogs := make([][]byte, 0) iter = reader.PrefixIterator([]byte("dog")) for iter.Valid() { // if we want to keep bytes from iteration we must copy k := iter.Key() copyk := make([]byte, len(k)) copy(copyk, k) dogs = append(dogs, copyk) if len(dogs) < 2 { iter.Seek([]byte("dog3")) } else { iter.Next() } } err = iter.Close() if err != nil { t.Fatal(err) } // check that we found the expected dogs if !reflect.DeepEqual(dogs, expectedDogs) { t.Fatalf("expected dogs %v, got %v", expectedDogs, dogs) } // close the reader err = reader.Close() if err != nil { t.Fatal(err) } }
func CommonTestRangeIteratorSeek(t *testing.T, s store.KVStore) { data := []testRow{ {[]byte("a1"), []byte("val")}, {[]byte("b1"), []byte("val")}, {[]byte("c1"), []byte("val")}, {[]byte("d1"), []byte("val")}, {[]byte("e1"), []byte("val")}, } err := batchWriteRows(s, data) if err != nil { t.Fatal(err) } // open a reader reader, err := s.Reader() if err != nil { t.Fatal(err) } // get an iterator on a central subset of the data start := []byte("b1") end := []byte("d1") iter := reader.RangeIterator(start, end) // seek before, at and after every possible key targets := [][]byte{} for _, row := range data { prefix := string(row.key[:1]) targets = append(targets, []byte(prefix+"0")) targets = append(targets, []byte(prefix+"1")) targets = append(targets, []byte(prefix+"2")) } for _, target := range targets { found := []string{} for iter.Seek(target); iter.Valid(); iter.Next() { found = append(found, string(iter.Key())) if len(found) > len(data) { t.Fatalf("enumerated more than data keys after seeking to %s", string(target)) } } wanted := []string{} for _, row := range data { if bytes.Compare(row.key, start) < 0 || bytes.Compare(row.key, target) < 0 || bytes.Compare(row.key, end) >= 0 { continue } wanted = append(wanted, string(row.key)) } fs := strings.Join(found, ", ") ws := strings.Join(wanted, ", ") if fs != ws { t.Fatalf("iterating from %s returned [%s] instead of [%s]", string(target), fs, ws) } } err = iter.Close() if err != nil { t.Fatal(err) } if err != nil { t.Fatal(err) } // close the reader err = reader.Close() if err != nil { t.Fatal(err) } }
func CommonTestRangeIterator(t *testing.T, s store.KVStore) { data := []testRow{ {[]byte("a1"), []byte("val")}, {[]byte("b1"), []byte("val")}, {[]byte("b2"), []byte("val")}, {[]byte("b3"), []byte("val")}, {[]byte("c1"), []byte("val")}, {[]byte("c2"), []byte("val")}, {[]byte("c4"), []byte("val")}, {[]byte("d1"), []byte("val")}, } expectedAll := make([][]byte, 0) expectedBToC := make([][]byte, 0) expectedCToDSeek3 := make([][]byte, 0) expectedCToEnd := make([][]byte, 0) for _, row := range data { expectedAll = append(expectedAll, row.key) if bytes.HasPrefix(row.key, []byte("b")) { expectedBToC = append(expectedBToC, row.key) } if bytes.HasPrefix(row.key, []byte("c")) && !bytes.HasSuffix(row.key, []byte("2")) { expectedCToDSeek3 = append(expectedCToDSeek3, row.key) } if bytes.Compare(row.key, []byte("c")) > 0 { expectedCToEnd = append(expectedCToEnd, row.key) } } err := batchWriteRows(s, data) if err != nil { t.Fatal(err) } // open a reader reader, err := s.Reader() if err != nil { t.Fatal(err) } // get a range iterator (all) all := make([][]byte, 0) iter := reader.RangeIterator(nil, nil) for iter.Valid() { // if we want to keep bytes from iteration we must copy k := iter.Key() copyk := make([]byte, len(k)) copy(copyk, k) all = append(all, copyk) iter.Next() } err = iter.Close() if err != nil { t.Fatal(err) } // check that we found all if !reflect.DeepEqual(all, expectedAll) { t.Fatalf("expected all %v, got %v", expectedAll, all) } // get range iterator from b - c bToC := make([][]byte, 0) iter = reader.RangeIterator([]byte("b"), []byte("c")) for iter.Valid() { // if we want to keep bytes from iteration we must copy k := iter.Key() copyk := make([]byte, len(k)) copy(copyk, k) bToC = append(bToC, copyk) iter.Next() } err = iter.Close() if err != nil { t.Fatal(err) } // check that we found b-c if !reflect.DeepEqual(bToC, expectedBToC) { t.Fatalf("expected b-c %v, got %v", expectedBToC, bToC) } // get range iterator from c - d, but seek to 'c3' cToDSeek3 := make([][]byte, 0) iter = reader.RangeIterator([]byte("c"), []byte("d")) for iter.Valid() { // if we want to keep bytes from iteration we must copy k := iter.Key() copyk := make([]byte, len(k)) copy(copyk, k) cToDSeek3 = append(cToDSeek3, copyk) if len(cToDSeek3) < 2 { iter.Seek([]byte("c3")) } else { iter.Next() } } err = iter.Close() if err != nil { t.Fatal(err) } // check that we found c-d with seek to c3 if !reflect.DeepEqual(cToDSeek3, expectedCToDSeek3) { t.Fatalf("expected b-c %v, got %v", expectedCToDSeek3, cToDSeek3) } // get range iterator from c to the end cToEnd := make([][]byte, 0) iter = reader.RangeIterator([]byte("c"), nil) for iter.Valid() { // if we want to keep bytes from iteration we must copy k := iter.Key() copyk := make([]byte, len(k)) copy(copyk, k) cToEnd = append(cToEnd, copyk) iter.Next() } err = iter.Close() if err != nil { t.Fatal(err) } // check that we found c to end if !reflect.DeepEqual(cToEnd, expectedCToEnd) { t.Fatalf("expected b-c %v, got %v", expectedCToEnd, cToEnd) } // close the reader err = reader.Close() if err != nil { t.Fatal(err) } }
func CommonTestPrefixIteratorSeek(t *testing.T, s store.KVStore) { data := []testRow{ {[]byte("a"), []byte("val")}, {[]byte("b1"), []byte("val")}, {[]byte("b2"), []byte("val")}, {[]byte("b3"), []byte("val")}, {[]byte("c"), []byte("val")}, } err := batchWriteRows(s, data) if err != nil { t.Fatal(err) } // open a reader reader, err := s.Reader() if err != nil { t.Fatal(err) } defer func() { err := reader.Close() if err != nil { t.Fatal(err) } }() // get an iterator on a central subset of the data iter := reader.PrefixIterator([]byte("b")) defer func() { err := iter.Close() if err != nil { t.Fatal(err) } }() // check that all keys have prefix found := []string{} for ; iter.Valid(); iter.Next() { found = append(found, string(iter.Key())) } for _, f := range found { if !strings.HasPrefix(f, "b") { t.Errorf("got key '%s' that doesn't have correct prefix", f) } } if len(found) != 3 { t.Errorf("expected 3 keys with prefix, got %d", len(found)) } // now try to seek before the prefix and repeat found = []string{} for iter.Seek([]byte("a")); iter.Valid(); iter.Next() { found = append(found, string(iter.Key())) } for _, f := range found { if !strings.HasPrefix(f, "b") { t.Errorf("got key '%s' that doesn't have correct prefix", f) } } if len(found) != 3 { t.Errorf("expected 3 keys with prefix, got %d", len(found)) } // now try to seek after the prefix and repeat found = []string{} for iter.Seek([]byte("c")); iter.Valid(); iter.Next() { found = append(found, string(iter.Key())) } for _, f := range found { if !strings.HasPrefix(f, "b") { t.Errorf("got key '%s' that doesn't have correct prefix", f) } } if len(found) != 0 { t.Errorf("expected 0 keys with prefix, got %d", len(found)) } }
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { // insert a kv pair writer, err := s.Writer() if err != nil { t.Error(err) } // ************************************************** // this is a hack only required for BoltDB // however its harmless so to keep the tests // the same everywhere, we include it here // // this is a hack to try to pre-emptively overflow // boltdb writes *MAY* block a long reader // in particular, if the write requires additional // allocation, it must acquire the same lock as // the reader, thus cannot continue until that // reader is closed. // in general this is not a problem for bleve // (though it may affect performance in some cases) // but it is a problem for this test which attempts // to easily verify that readers are isolated // this hack writes enough initial data such that // the subsequent writes do not require additional // space hackSize := 1000 batch := writer.NewBatch() for i := 0; i < hackSize; i++ { k := fmt.Sprintf("x%d", i) batch.Set([]byte(k), []byte("filler")) } err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } // ************************************************** batch = writer.NewBatch() batch.Set([]byte("a"), []byte("val-a")) err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } err = writer.Close() if err != nil { t.Fatal(err) } // create an isolated reader reader, err := s.Reader() if err != nil { t.Error(err) } defer func() { err := reader.Close() if err != nil { t.Fatal(err) } }() // verify that we see the value already inserted val, err := reader.Get([]byte("a")) if err != nil { t.Error(err) } if !reflect.DeepEqual(val, []byte("val-a")) { t.Errorf("expected val-a, got nil") } // verify that an iterator sees it count := 0 it := reader.RangeIterator([]byte{0}, []byte{'x'}) defer func() { err := it.Close() if err != nil { t.Fatal(err) } }() for it.Valid() { it.Next() count++ } if count != 1 { t.Errorf("expected iterator to see 1, saw %d", count) } // add something after the reader was created writer, err = s.Writer() if err != nil { t.Error(err) } batch = writer.NewBatch() batch.Set([]byte("b"), []byte("val-b")) err = writer.ExecuteBatch(batch) if err != nil { t.Fatal(err) } err = writer.Close() if err != nil { t.Fatal(err) } // ensure that a newer reader sees it newReader, err := s.Reader() if err != nil { t.Error(err) } defer func() { err := newReader.Close() if err != nil { t.Fatal(err) } }() val, err = newReader.Get([]byte("b")) if err != nil { t.Error(err) } if !reflect.DeepEqual(val, []byte("val-b")) { t.Errorf("expected val-b, got nil") } // ensure that the direct iterator sees it count = 0 it2 := newReader.RangeIterator([]byte{0}, []byte{'x'}) defer func() { err := it2.Close() if err != nil { t.Fatal(err) } }() for it2.Valid() { it2.Next() count++ } if count != 2 { t.Errorf("expected iterator to see 2, saw %d", count) } // but that the isolated reader does not val, err = reader.Get([]byte("b")) if err != nil { t.Error(err) } if val != nil { t.Errorf("expected nil, got %v", val) } // and ensure that the iterator on the isolated reader also does not count = 0 it3 := reader.RangeIterator([]byte{0}, []byte{'x'}) defer func() { err := it3.Close() if err != nil { t.Fatal(err) } }() for it3.Valid() { it3.Next() count++ } if count != 1 { t.Errorf("expected iterator to see 1, saw %d", count) } }