// Reindex rewrites the index files of the diskpacked .pack files func Reindex(root string, overwrite bool) (err error) { // there is newStorage, but that may open a file for writing var s = &storage{root: root} index, err := kvfile.NewStorage(filepath.Join(root, "index.kv")) if err != nil { return err } defer func() { closeErr := index.Close() // just returning the first error - if the index or disk is corrupt // and can't close, it's very likely these two errors are related and // have the same root cause. if err == nil { err = closeErr } }() verbose := false // TODO: use env var? for i := int64(0); i >= 0; i++ { fh, err := os.Open(s.filename(i)) if err != nil { if os.IsNotExist(err) { break } return err } err = reindexOne(index, overwrite, verbose, fh, fh.Name(), i) fh.Close() if err != nil { return err } } return nil }
// Reindex rewrites the index files of the diskpacked .pack files func Reindex(root string, overwrite bool) (err error) { // there is newStorage, but that may open a file for writing var s = &storage{root: root} index, err := kvfile.NewStorage(filepath.Join(root, "index.kv")) if err != nil { return err } defer func() { closeErr := index.Close() // just returning the first error - if the index or disk is corrupt // and can't close, it's very likely these two errors are related and // have the same root cause. if err == nil { err = closeErr } }() ctx := context.TODO() // TODO(tgulacsi): get the verbosity from context for i := 0; i >= 0; i++ { fh, err := os.Open(s.filename(i)) if err != nil { if os.IsNotExist(err) { break } return err } err = s.reindexOne(ctx, index, overwrite, i) fh.Close() if err != nil { return err } } return nil }
// Testing EnumerateBlobMeta because that's one of the few non-corpus index reading ops we still actually use. func BenchmarkEnumerateBlobMetaCznic(b *testing.B) { if *flagBenchDir == "" { b.Skip("Enumerating benchmark needs -benchDir") } dbfile := filepath.Join(*flagBenchDir, "kvfile.db") enumerateMeta(b, dbfile, func(dbfile string) (sorted.KeyValue, error) { return kvfile.NewStorage(dbfile) }) }
func BenchmarkInterruptCznic(b *testing.B) { if *flagBenchDir == "" { b.Skip("Interrupt benchmark needs -benchDir") } dbfile := filepath.Join(*flagBenchDir, "kvfile.db") // since cznic is much slower than levelDB at reindexing, we interrupt // it way less often. otherwise we might even blow up the max test run time // (10m) anyway. benchmarkKillReindex(b, 10, dbfile, func(dbfile string) (sorted.KeyValue, error) { return kvfile.NewStorage(dbfile) }) }
func newKvfileSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) { td, err := ioutil.TempDir("", "kvfile-test") if err != nil { t.Fatal(err) } kv, err = kvfile.NewStorage(filepath.Join(td, "kvfile")) if err != nil { os.RemoveAll(td) t.Fatal(err) } return kv, func() { kv.Close() os.RemoveAll(td) } }
// newStorage returns a new storage in path root with the given maxFileSize, // or defaultMaxFileSize (512MB) if <= 0 func newStorage(root string, maxFileSize int64, indexConf jsonconfig.Obj) (s *storage, err error) { fi, err := os.Stat(root) if os.IsNotExist(err) { return nil, fmt.Errorf("storage root %q doesn't exist", root) } if err != nil { return nil, fmt.Errorf("Failed to stat directory %q: %v", root, err) } if !fi.IsDir() { return nil, fmt.Errorf("storage root %q exists but is not a directory.", root) } var index sorted.KeyValue if len(indexConf) > 0 { index, err = sorted.NewKeyValue(indexConf) } else { index, err = kvfile.NewStorage(filepath.Join(root, indexKV)) } if err != nil { return nil, err } defer func() { if err != nil { index.Close() } }() if maxFileSize <= 0 { maxFileSize = defaultMaxFileSize } // Be consistent with trailing slashes. Makes expvar stats for total // reads/writes consistent across diskpacked targets, regardless of what // people put in their low level config. root = strings.TrimRight(root, `\/`) s = &storage{ root: root, index: index, maxFileSize: maxFileSize, Generationer: local.NewGenerationer(root), } s.mu.Lock() defer s.mu.Unlock() if err := s.openAllPacks(); err != nil { return nil, err } if _, _, err := s.StorageGeneration(); err != nil { return nil, fmt.Errorf("Error initialization generation for %q: %v", root, err) } return s, nil }
func TestChildIndexer(t *testing.T) { if os.Getenv("TEST_BE_CHILD") != "1" { t.Skip("not a real test; used as a child process by the benchmarks") } dbfile := os.Getenv("TEST_BE_CHILD_DBFILE") if dbfile == "" { log.Fatal("empty TEST_BE_CHILD_DBFILE") } if err := os.RemoveAll(dbfile); err != nil { log.Fatal(err) } var kv sorted.KeyValue var err error switch { case strings.HasSuffix(dbfile, "leveldb.db"): kv, err = leveldb.NewStorage(dbfile) case strings.HasSuffix(dbfile, "kvfile.db"): kv, err = kvfile.NewStorage(dbfile) case strings.HasSuffix(dbfile, "sqlite.db"): kv, err = sqlite.NewStorage(dbfile) default: log.Fatalf("unknown sorted provider for %v", dbfile) } if err != nil { log.Fatal(err) } bs, err := localdisk.New(filepath.Join(filepath.Dir(dbfile), "bs")) if err != nil { log.Fatal(err) } idx, err := index.New(kv) if err != nil { log.Fatal(err) } idx.InitBlobSource(bs) defer func() { if err := idx.Close(); err != nil { log.Fatal(err) } }() if err := idx.Reindex(); err != nil { log.Fatal(err) } }
// newStorage returns a new storage in path root with the given maxFileSize, // or defaultMaxFileSize (512MB) if <= 0 func newStorage(root string, maxFileSize int64) (s *storage, err error) { fi, err := os.Stat(root) if os.IsNotExist(err) { return nil, fmt.Errorf("storage root %q doesn't exist", root) } if err != nil { return nil, fmt.Errorf("Failed to stat directory %q: %v", root, err) } if !fi.IsDir() { return nil, fmt.Errorf("storage root %q exists but is not a directory.", root) } index, err := kvfile.NewStorage(filepath.Join(root, "index.kv")) if err != nil { return nil, err } defer func() { if err != nil { index.Close() } }() if maxFileSize <= 0 { maxFileSize = defaultMaxFileSize } s = &storage{ root: root, index: index, maxFileSize: maxFileSize, Generationer: local.NewGenerationer(root), } if err := s.openCurrent(); err != nil { return nil, err } if _, _, err := s.StorageGeneration(); err != nil { return nil, fmt.Errorf("Error initialization generation for %q: %v", root, err) } return s, nil }
func BenchmarkPopulateCznic(b *testing.B) { benchmarkPopulate(b, "kvfile.db", func(dbfile string) (sorted.KeyValue, error) { return kvfile.NewStorage(dbfile) }) }