func newTestStorage(t *testing.T, packs ...pack) (s *storage, clean func()) {
restoreLogging := test.TLog(t)
dir, err := ioutil.TempDir("", "diskpacked-test")
if err != nil {
t.Fatal(err)
}
for i, p := range packs {
writePack(t, dir, i, p)
}
if err := Reindex(dir, true, nil); err != nil {
t.Fatalf("Reindexing after writing pack files: %v", err)
}
s, err = newStorage(dir, 0, nil)
if err != nil {
t.Fatal(err)
}
clean = func() {
s.Close()
os.RemoveAll(dir)
restoreLogging()
}
return s, clean
}
func BenchmarkCorpusFromStorage(b *testing.B) {
defer test.TLog(b)()
buildKvOnce.Do(func() {
kvForBenchmark = sorted.NewMemoryKeyValue()
idx := index.New(kvForBenchmark)
id := indextest.NewIndexDeps(idx)
id.Fataler = b
for i := 0; i < 10; i++ {
fileRef, _ := id.UploadFile("file.txt", fmt.Sprintf("some file %d", i), time.Unix(1382073153, 0))
pn := id.NewPlannedPermanode(fmt.Sprint(i))
id.SetAttribute(pn, "camliContent", fileRef.String())
}
})
defer index.SetVerboseCorpusLogging(true)
index.SetVerboseCorpusLogging(false)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := index.NewCorpusFromStorage(kvForBenchmark)
if err != nil {
b.Fatal(err)
}
}
}
func testQueryTypes(t test.TB, types []indexType, fn func(*queryTest)) {
defer test.TLog(t)()
for _, it := range types {
if *queryType == "" || *queryType == it.String() {
t.Logf("Testing: --querytype=%s ...", it)
testQueryType(t, fn, it)
}
}
}
func TestSorted(t *testing.T, kv sorted.KeyValue) {
defer test.TLog(t)()
if !isEmpty(t, kv) {
t.Fatal("kv for test is expected to be initially empty")
}
set := func(k, v string) {
if err := kv.Set(k, v); err != nil {
t.Fatalf("Error setting %q to %q: %v", k, v, err)
}
}
set("foo", "bar")
if isEmpty(t, kv) {
t.Fatalf("iterator reports the kv is empty after adding foo=bar; iterator must be broken")
}
if v, err := kv.Get("foo"); err != nil || v != "bar" {
t.Errorf("get(foo) = %q, %v; want bar", v, err)
}
if v, err := kv.Get("NOT_EXIST"); err != sorted.ErrNotFound {
t.Errorf("get(NOT_EXIST) = %q, %v; want error sorted.ErrNotFound", v, err)
}
for i := 0; i < 2; i++ {
if err := kv.Delete("foo"); err != nil {
t.Errorf("Delete(foo) (on loop %d/2) returned error %v", i+1, err)
}
}
set("a", "av")
set("b", "bv")
set("c", "cv")
testEnumerate(t, kv, "", "", "av", "bv", "cv")
testEnumerate(t, kv, "a", "", "av", "bv", "cv")
testEnumerate(t, kv, "b", "", "bv", "cv")
testEnumerate(t, kv, "a", "c", "av", "bv")
testEnumerate(t, kv, "a", "b", "av")
testEnumerate(t, kv, "a", "a")
testEnumerate(t, kv, "d", "")
testEnumerate(t, kv, "d", "e")
// Verify that < comparison works identically for all DBs (because it is affected by collation rules)
// http://postgresql.1045698.n5.nabble.com/String-comparison-and-the-SQL-standard-td5740721.html
set("foo|abc", "foo|abcv")
testEnumerate(t, kv, "foo|", "", "foo|abcv")
testEnumerate(t, kv, "foo|", "foo}", "foo|abcv")
// Verify that the value isn't being used instead of the key in the range comparison.
set("y", "x:foo")
testEnumerate(t, kv, "x:", "x~")
testInsertLarge(t, kv)
testInsertTooLarge(t, kv)
// TODO: test batch commits
// Deleting a non-existent item in a batch should not be an error
testDeleteNotFoundBatch(t, kv)
testDeletePartialNotFoundBatch(t, kv)
}
func newShareTester(t *testing.T) *shareTester {
sto := new(test.Fetcher)
st := &shareTester{
t: t,
sto: sto,
handler: &shareHandler{fetcher: sto},
restoreLog: test.TLog(t),
}
timeSleep = func(d time.Duration) {
st.sleeps++
}
return st
}
func (tester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
defer test.TLog(t)()
var cleanups []func()
defer func() {
for _, fn := range cleanups {
fn()
}
}()
initIndex := func() *index.Index {
kv, cleanup := newSorted(t)
cleanups = append(cleanups, cleanup)
return index.MustNew(t, kv)
}
tfn(t, initIndex)
}
func TestSorted(t *testing.T, kv sorted.KeyValue) {
defer test.TLog(t)()
if !isEmpty(t, kv) {
t.Fatal("kv for test is expected to be initially empty")
}
set := func(k, v string) {
if err := kv.Set(k, v); err != nil {
t.Fatalf("Error setting %q to %q: %v", k, v, err)
}
}
set("foo", "bar")
if isEmpty(t, kv) {
t.Fatalf("iterator reports the kv is empty after adding foo=bar; iterator must be broken")
}
if v, err := kv.Get("foo"); err != nil || v != "bar" {
t.Errorf("get(foo) = %q, %v; want bar", v, err)
}
if v, err := kv.Get("NOT_EXIST"); err != sorted.ErrNotFound {
t.Errorf("get(NOT_EXIST) = %q, %v; want error sorted.ErrNotFound", v, err)
}
for i := 0; i < 2; i++ {
if err := kv.Delete("foo"); err != nil {
t.Errorf("Delete(foo) (on loop %d/2) returned error %v", i+1, err)
}
}
set("a", "av")
set("b", "bv")
set("c", "cv")
testEnumerate(t, kv, "", "", "av", "bv", "cv")
testEnumerate(t, kv, "a", "", "av", "bv", "cv")
testEnumerate(t, kv, "b", "", "bv", "cv")
testEnumerate(t, kv, "a", "c", "av", "bv")
testEnumerate(t, kv, "a", "b", "av")
testEnumerate(t, kv, "a", "a")
testEnumerate(t, kv, "d", "")
testEnumerate(t, kv, "d", "e")
// Verify that the value isn't being used instead of the key in the range comparison.
set("y", "x:foo")
testEnumerate(t, kv, "x:", "x~")
// TODO: test batch commits
}
func (mongoTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
defer test.TLog(t)()
var mu sync.Mutex // guards cleanups
var cleanups []func()
defer func() {
mu.Lock() // never unlocked
for _, fn := range cleanups {
fn()
}
}()
initIndex := func() *index.Index {
kv, cleanup := newSorted(t)
mu.Lock()
cleanups = append(cleanups, cleanup)
mu.Unlock()
return index.New(kv)
}
tfn(t, initIndex)
}
func indexTest(t *testing.T,
sortedGenfn func(t *testing.T) (sorted.KeyValue, func()),
tfn func(*testing.T, func() *index.Index)) {
defer test.TLog(t)()
var mu sync.Mutex // guards cleanups
var cleanups []func()
defer func() {
mu.Lock() // never unlocked
for _, fn := range cleanups {
fn()
}
}()
makeIndex := func() *index.Index {
s, cleanup := sortedGenfn(t)
mu.Lock()
cleanups = append(cleanups, cleanup)
mu.Unlock()
return indextest.MustNew(t, s)
}
tfn(t, makeIndex)
}
func newTempDiskpackedWithIndex(t *testing.T, indexConf jsonconfig.Obj) (sto blobserver.Storage, cleanup func()) {
restoreLogging := test.TLog(t)
dir, err := ioutil.TempDir("", "diskpacked-test")
if err != nil {
t.Fatal(err)
}
t.Logf("diskpacked test dir is %q", dir)
s, err := newStorage(dir, 1<<20, indexConf)
if err != nil {
t.Fatalf("newStorage: %v", err)
}
return s, func() {
s.Close()
if env.IsDebug() {
t.Logf("CAMLI_DEBUG set, skipping cleanup of dir %q", dir)
} else {
os.RemoveAll(dir)
}
restoreLogging()
}
}