Example #1
0
// Reindex rewrites the index files of the diskpacked .pack files
func Reindex(root string, overwrite bool) (err error) {
	// there is newStorage, but that may open a file for writing
	var s = &storage{root: root}
	index, err := kvfile.NewStorage(filepath.Join(root, "index.kv"))
	if err != nil {
		return err
	}
	defer func() {
		closeErr := index.Close()
		// just returning the first error - if the index or disk is corrupt
		// and can't close, it's very likely these two errors are related and
		// have the same root cause.
		if err == nil {
			err = closeErr
		}
	}()

	verbose := false // TODO: use env var?
	for i := int64(0); i >= 0; i++ {
		fh, err := os.Open(s.filename(i))
		if err != nil {
			if os.IsNotExist(err) {
				break
			}
			return err
		}
		err = reindexOne(index, overwrite, verbose, fh, fh.Name(), i)
		fh.Close()
		if err != nil {
			return err
		}
	}
	return nil
}
Example #2
0
// Reindex rewrites the index files of the diskpacked .pack files
func Reindex(root string, overwrite bool) (err error) {
	// there is newStorage, but that may open a file for writing
	var s = &storage{root: root}
	index, err := kvfile.NewStorage(filepath.Join(root, "index.kv"))
	if err != nil {
		return err
	}
	defer func() {
		closeErr := index.Close()
		// just returning the first error - if the index or disk is corrupt
		// and can't close, it's very likely these two errors are related and
		// have the same root cause.
		if err == nil {
			err = closeErr
		}
	}()

	ctx := context.TODO() // TODO(tgulacsi): get the verbosity from context
	for i := 0; i >= 0; i++ {
		fh, err := os.Open(s.filename(i))
		if err != nil {
			if os.IsNotExist(err) {
				break
			}
			return err
		}
		err = s.reindexOne(ctx, index, overwrite, i)
		fh.Close()
		if err != nil {
			return err
		}
	}
	return nil
}
Example #3
0
// Testing EnumerateBlobMeta because that's one of the few non-corpus index reading ops we still actually use.
func BenchmarkEnumerateBlobMetaCznic(b *testing.B) {
	if *flagBenchDir == "" {
		b.Skip("Enumerating benchmark needs -benchDir")
	}
	dbfile := filepath.Join(*flagBenchDir, "kvfile.db")
	enumerateMeta(b, dbfile, func(dbfile string) (sorted.KeyValue, error) {
		return kvfile.NewStorage(dbfile)
	})
}
Example #4
0
func BenchmarkInterruptCznic(b *testing.B) {
	if *flagBenchDir == "" {
		b.Skip("Interrupt benchmark needs -benchDir")
	}
	dbfile := filepath.Join(*flagBenchDir, "kvfile.db")

	// since cznic is much slower than levelDB at reindexing, we interrupt
	// it way less often. otherwise we might even blow up the max test run time
	// (10m) anyway.
	benchmarkKillReindex(b, 10, dbfile, func(dbfile string) (sorted.KeyValue, error) {
		return kvfile.NewStorage(dbfile)
	})
}
Example #5
0
func newKvfileSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
	td, err := ioutil.TempDir("", "kvfile-test")
	if err != nil {
		t.Fatal(err)
	}
	kv, err = kvfile.NewStorage(filepath.Join(td, "kvfile"))
	if err != nil {
		os.RemoveAll(td)
		t.Fatal(err)
	}
	return kv, func() {
		kv.Close()
		os.RemoveAll(td)
	}
}
Example #6
0
// newStorage returns a new storage in path root with the given maxFileSize,
// or defaultMaxFileSize (512MB) if <= 0
func newStorage(root string, maxFileSize int64, indexConf jsonconfig.Obj) (s *storage, err error) {
	fi, err := os.Stat(root)
	if os.IsNotExist(err) {
		return nil, fmt.Errorf("storage root %q doesn't exist", root)
	}
	if err != nil {
		return nil, fmt.Errorf("Failed to stat directory %q: %v", root, err)
	}
	if !fi.IsDir() {
		return nil, fmt.Errorf("storage root %q exists but is not a directory.", root)
	}
	var index sorted.KeyValue
	if len(indexConf) > 0 {
		index, err = sorted.NewKeyValue(indexConf)
	} else {
		index, err = kvfile.NewStorage(filepath.Join(root, indexKV))
	}
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			index.Close()
		}
	}()
	if maxFileSize <= 0 {
		maxFileSize = defaultMaxFileSize
	}
	// Be consistent with trailing slashes.  Makes expvar stats for total
	// reads/writes consistent across diskpacked targets, regardless of what
	// people put in their low level config.
	root = strings.TrimRight(root, `\/`)
	s = &storage{
		root:         root,
		index:        index,
		maxFileSize:  maxFileSize,
		Generationer: local.NewGenerationer(root),
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if err := s.openAllPacks(); err != nil {
		return nil, err
	}
	if _, _, err := s.StorageGeneration(); err != nil {
		return nil, fmt.Errorf("Error initialization generation for %q: %v", root, err)
	}
	return s, nil
}
Example #7
0
func TestChildIndexer(t *testing.T) {
	if os.Getenv("TEST_BE_CHILD") != "1" {
		t.Skip("not a real test; used as a child process by the benchmarks")
	}
	dbfile := os.Getenv("TEST_BE_CHILD_DBFILE")
	if dbfile == "" {
		log.Fatal("empty TEST_BE_CHILD_DBFILE")
	}
	if err := os.RemoveAll(dbfile); err != nil {
		log.Fatal(err)
	}
	var kv sorted.KeyValue
	var err error
	switch {
	case strings.HasSuffix(dbfile, "leveldb.db"):
		kv, err = leveldb.NewStorage(dbfile)
	case strings.HasSuffix(dbfile, "kvfile.db"):
		kv, err = kvfile.NewStorage(dbfile)
	case strings.HasSuffix(dbfile, "sqlite.db"):
		kv, err = sqlite.NewStorage(dbfile)
	default:
		log.Fatalf("unknown sorted provider for %v", dbfile)
	}
	if err != nil {
		log.Fatal(err)
	}
	bs, err := localdisk.New(filepath.Join(filepath.Dir(dbfile), "bs"))
	if err != nil {
		log.Fatal(err)
	}
	idx, err := index.New(kv)
	if err != nil {
		log.Fatal(err)
	}
	idx.InitBlobSource(bs)
	defer func() {
		if err := idx.Close(); err != nil {
			log.Fatal(err)
		}
	}()
	if err := idx.Reindex(); err != nil {
		log.Fatal(err)
	}
}
Example #8
0
// newStorage returns a new storage in path root with the given maxFileSize,
// or defaultMaxFileSize (512MB) if <= 0
func newStorage(root string, maxFileSize int64) (s *storage, err error) {
	fi, err := os.Stat(root)
	if os.IsNotExist(err) {
		return nil, fmt.Errorf("storage root %q doesn't exist", root)
	}
	if err != nil {
		return nil, fmt.Errorf("Failed to stat directory %q: %v", root, err)
	}
	if !fi.IsDir() {
		return nil, fmt.Errorf("storage root %q exists but is not a directory.", root)
	}
	index, err := kvfile.NewStorage(filepath.Join(root, "index.kv"))
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			index.Close()
		}
	}()
	if maxFileSize <= 0 {
		maxFileSize = defaultMaxFileSize
	}
	s = &storage{
		root:         root,
		index:        index,
		maxFileSize:  maxFileSize,
		Generationer: local.NewGenerationer(root),
	}
	if err := s.openCurrent(); err != nil {
		return nil, err
	}
	if _, _, err := s.StorageGeneration(); err != nil {
		return nil, fmt.Errorf("Error initialization generation for %q: %v", root, err)
	}
	return s, nil
}
Example #9
0
func BenchmarkPopulateCznic(b *testing.B) {
	benchmarkPopulate(b, "kvfile.db", func(dbfile string) (sorted.KeyValue, error) {
		return kvfile.NewStorage(dbfile)
	})
}