// Testing EnumerateBlobMeta because that's one of the few non-corpus index reading ops we still actually use.
func BenchmarkEnumerateBlobMetaLevelDB(b *testing.B) {
if *flagBenchDir == "" {
b.Skip("Enumerating benchmark needs -benchDir")
}
dbfile := filepath.Join(*flagBenchDir, "leveldb.db")
enumerateMeta(b, dbfile, func(dbfile string) (sorted.KeyValue, error) {
return leveldb.NewStorage(dbfile)
})
}
func BenchmarkInterruptLevelDB(b *testing.B) {
if *flagBenchDir == "" {
b.Skip("Interrupt benchmark needs -benchDir")
}
dbfile := filepath.Join(*flagBenchDir, "leveldb.db")
benchmarkKillReindex(b, 1, dbfile, func(dbfile string) (sorted.KeyValue, error) {
return leveldb.NewStorage(dbfile)
})
}
func newLevelDBSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
td, err := ioutil.TempDir("", "camli-index-leveldb")
if err != nil {
t.Fatal(err)
}
kv, err = leveldb.NewStorage(filepath.Join(td, "leveldb"))
if err != nil {
os.RemoveAll(td)
t.Fatal(err)
}
return kv, func() {
kv.Close()
os.RemoveAll(td)
}
}
func TestChildIndexer(t *testing.T) {
if os.Getenv("TEST_BE_CHILD") != "1" {
t.Skip("not a real test; used as a child process by the benchmarks")
}
dbfile := os.Getenv("TEST_BE_CHILD_DBFILE")
if dbfile == "" {
log.Fatal("empty TEST_BE_CHILD_DBFILE")
}
if err := os.RemoveAll(dbfile); err != nil {
log.Fatal(err)
}
var kv sorted.KeyValue
var err error
switch {
case strings.HasSuffix(dbfile, "leveldb.db"):
kv, err = leveldb.NewStorage(dbfile)
case strings.HasSuffix(dbfile, "kvfile.db"):
kv, err = kvfile.NewStorage(dbfile)
case strings.HasSuffix(dbfile, "sqlite.db"):
kv, err = sqlite.NewStorage(dbfile)
default:
log.Fatalf("unknown sorted provider for %v", dbfile)
}
if err != nil {
log.Fatal(err)
}
bs, err := localdisk.New(filepath.Join(filepath.Dir(dbfile), "bs"))
if err != nil {
log.Fatal(err)
}
idx, err := index.New(kv)
if err != nil {
log.Fatal(err)
}
idx.InitBlobSource(bs)
defer func() {
if err := idx.Close(); err != nil {
log.Fatal(err)
}
}()
if err := idx.Reindex(); err != nil {
log.Fatal(err)
}
}
func main() {
flag.Parse()
if len(*blobDir)*len(*indexDir) == 0 {
flag.Usage()
return
}
s, err := dir.New(*blobDir)
if err != nil {
log.Fatal(err)
}
src, ok := s.(blobserver.BlobStreamer)
if !ok {
log.Fatalf("%v is not a BlobStreamer", s)
}
db, err := leveldb.NewStorage(*indexDir)
if err != nil {
log.Fatal(err)
}
defer db.Close()
dst, err := index.New(db)
if err != nil {
log.Fatal(err)
}
fe := FetcherEnumerator{
FetcherEnumerator: s,
c1: make(map[string]*blob.Blob),
start: time.Now(),
}
dst.InitBlobSource(&fe)
ch := make(chan blobserver.BlobAndToken)
go fe.PrintStats()
for i := 0; i < *parallel; i++ {
go fe.Index(ch, dst)
}
ctx := context.New()
if err := src.StreamBlobs(ctx, ch, *streamStart); err != nil {
log.Fatal(err)
}
}
// dataStores returns the blobserver that stores the instances configurations, and the kv
// store for the instances states.
func dataStores() (blobserver.Storage, sorted.KeyValue, error) {
dataDir := os.Getenv("CAMLI_GCE_DATA")
if dataDir == "" {
dataDir = "camli-gce-data"
log.Printf("data dir not provided as env var CAMLI_GCE_DATA, so defaulting to %v", dataDir)
}
blobsDir := filepath.Join(dataDir, "instance-conf")
if err := os.MkdirAll(blobsDir, 0700); err != nil {
return nil, nil, err
}
instConf, err := localdisk.New(blobsDir)
if err != nil {
return nil, nil, err
}
instState, err := leveldb.NewStorage(filepath.Join(dataDir, "instance-state"))
if err != nil {
return nil, nil, err
}
return instConf, instState, nil
}
func BenchmarkPopulateLevelDB(b *testing.B) {
benchmarkPopulate(b, "leveldb.db", func(dbfile string) (sorted.KeyValue, error) {
return leveldb.NewStorage(dbfile)
})
}
