// Testing EnumerateBlobMeta because that's one of the few non-corpus index reading ops we still actually use. func BenchmarkEnumerateBlobMetaLevelDB(b *testing.B) { if *flagBenchDir == "" { b.Skip("Enumerating benchmark needs -benchDir") } dbfile := filepath.Join(*flagBenchDir, "leveldb.db") enumerateMeta(b, dbfile, func(dbfile string) (sorted.KeyValue, error) { return leveldb.NewStorage(dbfile) }) }
func BenchmarkInterruptLevelDB(b *testing.B) { if *flagBenchDir == "" { b.Skip("Interrupt benchmark needs -benchDir") } dbfile := filepath.Join(*flagBenchDir, "leveldb.db") benchmarkKillReindex(b, 1, dbfile, func(dbfile string) (sorted.KeyValue, error) { return leveldb.NewStorage(dbfile) }) }
func newLevelDBSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) { td, err := ioutil.TempDir("", "camli-index-leveldb") if err != nil { t.Fatal(err) } kv, err = leveldb.NewStorage(filepath.Join(td, "leveldb")) if err != nil { os.RemoveAll(td) t.Fatal(err) } return kv, func() { kv.Close() os.RemoveAll(td) } }
func TestChildIndexer(t *testing.T) { if os.Getenv("TEST_BE_CHILD") != "1" { t.Skip("not a real test; used as a child process by the benchmarks") } dbfile := os.Getenv("TEST_BE_CHILD_DBFILE") if dbfile == "" { log.Fatal("empty TEST_BE_CHILD_DBFILE") } if err := os.RemoveAll(dbfile); err != nil { log.Fatal(err) } var kv sorted.KeyValue var err error switch { case strings.HasSuffix(dbfile, "leveldb.db"): kv, err = leveldb.NewStorage(dbfile) case strings.HasSuffix(dbfile, "kvfile.db"): kv, err = kvfile.NewStorage(dbfile) case strings.HasSuffix(dbfile, "sqlite.db"): kv, err = sqlite.NewStorage(dbfile) default: log.Fatalf("unknown sorted provider for %v", dbfile) } if err != nil { log.Fatal(err) } bs, err := localdisk.New(filepath.Join(filepath.Dir(dbfile), "bs")) if err != nil { log.Fatal(err) } idx, err := index.New(kv) if err != nil { log.Fatal(err) } idx.InitBlobSource(bs) defer func() { if err := idx.Close(); err != nil { log.Fatal(err) } }() if err := idx.Reindex(); err != nil { log.Fatal(err) } }
func main() { flag.Parse() if len(*blobDir)*len(*indexDir) == 0 { flag.Usage() return } s, err := dir.New(*blobDir) if err != nil { log.Fatal(err) } src, ok := s.(blobserver.BlobStreamer) if !ok { log.Fatalf("%v is not a BlobStreamer", s) } db, err := leveldb.NewStorage(*indexDir) if err != nil { log.Fatal(err) } defer db.Close() dst, err := index.New(db) if err != nil { log.Fatal(err) } fe := FetcherEnumerator{ FetcherEnumerator: s, c1: make(map[string]*blob.Blob), start: time.Now(), } dst.InitBlobSource(&fe) ch := make(chan blobserver.BlobAndToken) go fe.PrintStats() for i := 0; i < *parallel; i++ { go fe.Index(ch, dst) } ctx := context.New() if err := src.StreamBlobs(ctx, ch, *streamStart); err != nil { log.Fatal(err) } }
// dataStores returns the blobserver that stores the instances configurations, and the kv // store for the instances states. func dataStores() (blobserver.Storage, sorted.KeyValue, error) { dataDir := os.Getenv("CAMLI_GCE_DATA") if dataDir == "" { dataDir = "camli-gce-data" log.Printf("data dir not provided as env var CAMLI_GCE_DATA, so defaulting to %v", dataDir) } blobsDir := filepath.Join(dataDir, "instance-conf") if err := os.MkdirAll(blobsDir, 0700); err != nil { return nil, nil, err } instConf, err := localdisk.New(blobsDir) if err != nil { return nil, nil, err } instState, err := leveldb.NewStorage(filepath.Join(dataDir, "instance-state")) if err != nil { return nil, nil, err } return instConf, instState, nil }
func BenchmarkPopulateLevelDB(b *testing.B) { benchmarkPopulate(b, "leveldb.db", func(dbfile string) (sorted.KeyValue, error) { return leveldb.NewStorage(dbfile) }) }