Beispiel #1
0
func newTestStorage(t *testing.T, packs ...pack) (s *storage, clean func()) {
	restoreLogging := test.TLog(t)
	dir, err := ioutil.TempDir("", "diskpacked-test")
	if err != nil {
		t.Fatal(err)
	}

	for i, p := range packs {
		writePack(t, dir, i, p)
	}

	if err := Reindex(dir, true, nil); err != nil {
		t.Fatalf("Reindexing after writing pack files: %v", err)
	}
	s, err = newStorage(dir, 0, nil)
	if err != nil {
		t.Fatal(err)
	}

	clean = func() {
		s.Close()
		os.RemoveAll(dir)
		restoreLogging()
	}
	return s, clean
}
func BenchmarkCorpusFromStorage(b *testing.B) {
	defer test.TLog(b)()
	buildKvOnce.Do(func() {
		kvForBenchmark = sorted.NewMemoryKeyValue()
		idx := index.New(kvForBenchmark)
		id := indextest.NewIndexDeps(idx)
		id.Fataler = b
		for i := 0; i < 10; i++ {
			fileRef, _ := id.UploadFile("file.txt", fmt.Sprintf("some file %d", i), time.Unix(1382073153, 0))
			pn := id.NewPlannedPermanode(fmt.Sprint(i))
			id.SetAttribute(pn, "camliContent", fileRef.String())
		}
	})
	defer index.SetVerboseCorpusLogging(true)
	index.SetVerboseCorpusLogging(false)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, err := index.NewCorpusFromStorage(kvForBenchmark)
		if err != nil {
			b.Fatal(err)
		}
	}
}
Beispiel #3
0
func testQueryTypes(t test.TB, types []indexType, fn func(*queryTest)) {
	defer test.TLog(t)()
	for _, it := range types {
		if *queryType == "" || *queryType == it.String() {
			t.Logf("Testing: --querytype=%s ...", it)
			testQueryType(t, fn, it)
		}
	}
}
Beispiel #4
0
func TestSorted(t *testing.T, kv sorted.KeyValue) {
	defer test.TLog(t)()
	if !isEmpty(t, kv) {
		t.Fatal("kv for test is expected to be initially empty")
	}
	set := func(k, v string) {
		if err := kv.Set(k, v); err != nil {
			t.Fatalf("Error setting %q to %q: %v", k, v, err)
		}
	}
	set("foo", "bar")
	if isEmpty(t, kv) {
		t.Fatalf("iterator reports the kv is empty after adding foo=bar; iterator must be broken")
	}
	if v, err := kv.Get("foo"); err != nil || v != "bar" {
		t.Errorf("get(foo) = %q, %v; want bar", v, err)
	}
	if v, err := kv.Get("NOT_EXIST"); err != sorted.ErrNotFound {
		t.Errorf("get(NOT_EXIST) = %q, %v; want error sorted.ErrNotFound", v, err)
	}
	for i := 0; i < 2; i++ {
		if err := kv.Delete("foo"); err != nil {
			t.Errorf("Delete(foo) (on loop %d/2) returned error %v", i+1, err)
		}
	}
	set("a", "av")
	set("b", "bv")
	set("c", "cv")
	testEnumerate(t, kv, "", "", "av", "bv", "cv")
	testEnumerate(t, kv, "a", "", "av", "bv", "cv")
	testEnumerate(t, kv, "b", "", "bv", "cv")
	testEnumerate(t, kv, "a", "c", "av", "bv")
	testEnumerate(t, kv, "a", "b", "av")
	testEnumerate(t, kv, "a", "a")
	testEnumerate(t, kv, "d", "")
	testEnumerate(t, kv, "d", "e")

	// Verify that < comparison works identically for all DBs (because it is affected by collation rules)
	// http://postgresql.1045698.n5.nabble.com/String-comparison-and-the-SQL-standard-td5740721.html
	set("foo|abc", "foo|abcv")
	testEnumerate(t, kv, "foo|", "", "foo|abcv")
	testEnumerate(t, kv, "foo|", "foo}", "foo|abcv")

	// Verify that the value isn't being used instead of the key in the range comparison.
	set("y", "x:foo")
	testEnumerate(t, kv, "x:", "x~")

	testInsertLarge(t, kv)
	testInsertTooLarge(t, kv)

	// TODO: test batch commits

	// Deleting a non-existent item in a batch should not be an error
	testDeleteNotFoundBatch(t, kv)
	testDeletePartialNotFoundBatch(t, kv)
}
Beispiel #5
0
func newShareTester(t *testing.T) *shareTester {
	sto := new(test.Fetcher)
	st := &shareTester{
		t:          t,
		sto:        sto,
		handler:    &shareHandler{fetcher: sto},
		restoreLog: test.TLog(t),
	}
	timeSleep = func(d time.Duration) {
		st.sleeps++
	}
	return st
}
Beispiel #6
0
func (tester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
	defer test.TLog(t)()
	var cleanups []func()
	defer func() {
		for _, fn := range cleanups {
			fn()
		}
	}()

	initIndex := func() *index.Index {
		kv, cleanup := newSorted(t)
		cleanups = append(cleanups, cleanup)
		return index.MustNew(t, kv)
	}

	tfn(t, initIndex)
}
Beispiel #7
0
func TestSorted(t *testing.T, kv sorted.KeyValue) {
	defer test.TLog(t)()
	if !isEmpty(t, kv) {
		t.Fatal("kv for test is expected to be initially empty")
	}
	set := func(k, v string) {
		if err := kv.Set(k, v); err != nil {
			t.Fatalf("Error setting %q to %q: %v", k, v, err)
		}
	}
	set("foo", "bar")
	if isEmpty(t, kv) {
		t.Fatalf("iterator reports the kv is empty after adding foo=bar; iterator must be broken")
	}
	if v, err := kv.Get("foo"); err != nil || v != "bar" {
		t.Errorf("get(foo) = %q, %v; want bar", v, err)
	}
	if v, err := kv.Get("NOT_EXIST"); err != sorted.ErrNotFound {
		t.Errorf("get(NOT_EXIST) = %q, %v; want error sorted.ErrNotFound", v, err)
	}
	for i := 0; i < 2; i++ {
		if err := kv.Delete("foo"); err != nil {
			t.Errorf("Delete(foo) (on loop %d/2) returned error %v", i+1, err)
		}
	}
	set("a", "av")
	set("b", "bv")
	set("c", "cv")
	testEnumerate(t, kv, "", "", "av", "bv", "cv")
	testEnumerate(t, kv, "a", "", "av", "bv", "cv")
	testEnumerate(t, kv, "b", "", "bv", "cv")
	testEnumerate(t, kv, "a", "c", "av", "bv")
	testEnumerate(t, kv, "a", "b", "av")
	testEnumerate(t, kv, "a", "a")
	testEnumerate(t, kv, "d", "")
	testEnumerate(t, kv, "d", "e")

	// Verify that the value isn't being used instead of the key in the range comparison.
	set("y", "x:foo")
	testEnumerate(t, kv, "x:", "x~")

	// TODO: test batch commits
}
Beispiel #8
0
func (mongoTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
	defer test.TLog(t)()
	var mu sync.Mutex // guards cleanups
	var cleanups []func()
	defer func() {
		mu.Lock() // never unlocked
		for _, fn := range cleanups {
			fn()
		}
	}()
	initIndex := func() *index.Index {
		kv, cleanup := newSorted(t)
		mu.Lock()
		cleanups = append(cleanups, cleanup)
		mu.Unlock()
		return index.New(kv)
	}
	tfn(t, initIndex)
}
Beispiel #9
0
func indexTest(t *testing.T,
	sortedGenfn func(t *testing.T) (sorted.KeyValue, func()),
	tfn func(*testing.T, func() *index.Index)) {
	defer test.TLog(t)()
	var mu sync.Mutex // guards cleanups
	var cleanups []func()
	defer func() {
		mu.Lock() // never unlocked
		for _, fn := range cleanups {
			fn()
		}
	}()
	makeIndex := func() *index.Index {
		s, cleanup := sortedGenfn(t)
		mu.Lock()
		cleanups = append(cleanups, cleanup)
		mu.Unlock()
		return indextest.MustNew(t, s)
	}
	tfn(t, makeIndex)
}
func newTempDiskpackedWithIndex(t *testing.T, indexConf jsonconfig.Obj) (sto blobserver.Storage, cleanup func()) {
	restoreLogging := test.TLog(t)
	dir, err := ioutil.TempDir("", "diskpacked-test")
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("diskpacked test dir is %q", dir)
	s, err := newStorage(dir, 1<<20, indexConf)
	if err != nil {
		t.Fatalf("newStorage: %v", err)
	}
	return s, func() {
		s.Close()
		if env.IsDebug() {
			t.Logf("CAMLI_DEBUG set, skipping cleanup of dir %q", dir)
		} else {
			os.RemoveAll(dir)
		}
		restoreLogging()
	}
}