Exemplo n.º 1
0
// NewMDServerLocal constructs a new MDServerLocal object that stores
// data in the directories specified as parameters to this function.
func NewMDServerLocal(config Config, handleDbfile string, mdDbfile string,
	branchDbfile string) (*MDServerLocal, error) {

	handleStorage, err := storage.OpenFile(handleDbfile)
	if err != nil {
		return nil, err
	}

	mdStorage, err := storage.OpenFile(mdDbfile)
	if err != nil {
		return nil, err
	}

	branchStorage, err := storage.OpenFile(branchDbfile)
	if err != nil {
		return nil, err
	}

	// Always use memory for the lock storage, so it gets wiped after
	// a restart.
	lockStorage := storage.NewMemStorage()

	return newMDServerLocalWithStorage(config, handleStorage, mdStorage,
		branchStorage, lockStorage)
}
Exemplo n.º 2
0
func openDBBench(b *testing.B) *dbBench {
	_, err := os.Stat(benchDB)
	if err == nil {
		err = os.RemoveAll(benchDB)
		if err != nil {
			b.Fatal("cannot remove old db: ", err)
		}
	}

	p := &dbBench{b: b}
	p.stor, err = storage.OpenFile(benchDB)
	if err != nil {
		b.Fatal("cannot open stor: ", err)
	}

	p.o = &opt.Options{
		Flag: opt.OFCreateIfMissing,
	}
	p.ro = &opt.ReadOptions{}
	p.wo = &opt.WriteOptions{}

	p.db, err = Open(p.stor, p.o)
	if err != nil {
		b.Fatal("cannot open db: ", err)
	}

	runtime.GOMAXPROCS(runtime.NumCPU())
	return p
}
Exemplo n.º 3
0
func openDBBench(b *testing.B, noCompress bool) *dbBench {
	_, err := os.Stat(benchDB)
	if err == nil {
		err = os.RemoveAll(benchDB)
		if err != nil {
			b.Fatal("cannot remove old db: ", err)
		}
	}

	p := &dbBench{
		b:  b,
		o:  &opt.Options{},
		ro: &opt.ReadOptions{},
		wo: &opt.WriteOptions{},
	}
	p.stor, err = storage.OpenFile(benchDB, false)
	if err != nil {
		b.Fatal("cannot open stor: ", err)
	}
	if noCompress {
		p.o.Compression = opt.NoCompression
	}

	p.db, err = Open(p.stor, p.o)
	if err != nil {
		b.Fatal("cannot open db: ", err)
	}

	return p
}
Exemplo n.º 4
0
// Open will open and possibly create a datastore at the given directory.
func OpenLeveldb(path string, create bool, kvOpts Options) (db Engine, err error) {
	goOpts := kvOpts.(*goKeyValueOptions)
	if goOpts == nil {
		err = fmt.Errorf("Nil pointer passed in as key-value options to Openleveldb()!")
		return
	}

	leveldb_stor, err := storage.OpenFile(path)
	if err != nil {
		return
	}

	// Set the CreateIfMissing flag.
	if create {
		goOpts.Options.Flag |= opt.OFCreateIfMissing
		goOpts.Options.Flag |= opt.OFErrorIfExist
	}

	// Open the leveldb
	leveldb_db, err := leveldb.Open(leveldb_stor, goOpts.Options)
	if err != nil {
		return
	}
	db = &goLDB{
		directory: path,
		opts:      *goOpts, // We want a copy at time of Open()
		stor:      leveldb_stor,
		ldb:       leveldb_db,
	}
	return
}
Exemplo n.º 5
0
func TestDb_CreateReopenDbOnFile(t *testing.T) {
	dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile-%d", os.Getuid()))
	if err := os.RemoveAll(dbpath); err != nil {
		t.Fatal("cannot remove old db: ", err)
	}
	defer os.RemoveAll(dbpath)

	for i := 0; i < 3; i++ {
		stor, err := storage.OpenFile(dbpath)
		if err != nil {
			t.Fatalf("(%d) cannot open storage: %s", i, err)
		}
		db, err := Open(stor, &opt.Options{Flag: opt.OFCreateIfMissing})
		if err != nil {
			t.Fatalf("(%d) cannot open db: %s", i, err)
		}
		if err := db.Put([]byte("foo"), []byte("bar"), &opt.WriteOptions{}); err != nil {
			t.Fatalf("(%d) cannot write to db: %s", i, err)
		}
		if err := db.Close(); err != nil {
			t.Fatalf("(%d) cannot close db: %s", i, err)
		}
		if err := stor.Close(); err != nil {
			t.Fatalf("(%d) cannot close storage: %s", i, err)
		}
	}
}
Exemplo n.º 6
0
func NewStorage() *Storage {
	var stor storage.Storage
	var closeFn func() error
	if storageUseFS {
		for {
			storageMu.Lock()
			num := storageNum
			storageNum++
			storageMu.Unlock()
			path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
			if _, err := os.Stat(path); os.IsNotExist(err) {
				stor, err = storage.OpenFile(path)
				ExpectWithOffset(1, err).NotTo(HaveOccurred(), "creating storage at %s", path)
				closeFn = func() error {
					if storageKeepFS {
						return nil
					}
					return os.RemoveAll(path)
				}
				break
			}
		}
	} else {
		stor = storage.NewMemStorage()
	}
	s := &Storage{
		Storage: stor,
		closeFn: closeFn,
		opens:   make(map[uint64]bool),
	}
	s.stallCond.L = &s.mu
	return s
}
Exemplo n.º 7
0
// Main method. Will panic if things are so bad that the application
// will not start.
func main() {
	flag.Parse()

	log.Println("Event store to use:", *eventStorePath)
	log.Println("Command socket path:", *commandSocketZPath)
	log.Println("Event publishing socket path:", *eventPublishZPath)
	log.Println()

	var stor storage.Storage
	if *inMemoryStore {
		log.Println("!!! WARNING: Using in-memory store.")
		log.Println("!!! Events will not be persisted.")
		log.Println()
		stor = &storage.MemStorage{}
	} else {
		stor, err := storage.OpenFile(*eventStorePath)
		if err != nil {
			log.Panicln("could not create DB storage")
		}
		defer stor.Close()
	}

	estore, err := eventstore.New(stor)
	if err != nil {
		log.Panicln(os.Stderr, "could not create event store")
	}

	context, err := zmq.NewContext()
	if err != nil {
		log.Panicln(err)
	}

	initParams := server.InitParams{
		Store:              estore,
		CommandSocketZPath: commandSocketZPath,
		EvPubSocketZPath:   eventPublishZPath,
		ZMQContext:         context,
	}
	serv, err := server.New(&initParams)
	if err != nil {
		panic(err.Error())
	}

	sigchan := make(chan os.Signal, 5)
	serverStopper := func() {
		sig := <-sigchan
		if sig == os.Interrupt {
			serv.Stop()
		}
	}
	go serverStopper()
	signal.Notify(sigchan)

	serv.Start()
	serv.Wait()
}
Exemplo n.º 8
0
func (ulevel) OpenDatabase(name string, o level.UnderlyingOptions) (dtb level.UnderlyingDatabase, err error) {
	stor, err := storage.OpenFile(name)
	if err != nil {
		return
	}
	var dtbe *leveldb.DB
	dtbe, err = leveldb.Open(stor, o.(opts).Options)
	dtb = db{dtbe, stor}
	return
}
Exemplo n.º 9
0
func newTestStorage(t *testing.T) *testStorage {
	var stor storage.Storage
	var closeFn func() error
	if tsFS {
		for {
			tsMU.Lock()
			num := tsNum
			tsNum++
			tsMU.Unlock()
			path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
			if _, err := os.Stat(path); err != nil {
				stor, err = storage.OpenFile(path)
				if err != nil {
					t.Fatalf("F: cannot create storage: %v", err)
				}
				t.Logf("I: storage created: %s", path)
				closeFn = func() error {
					for _, name := range []string{"LOG.old", "LOG"} {
						f, err := os.Open(filepath.Join(path, name))
						if err != nil {
							continue
						}
						if log, err := ioutil.ReadAll(f); err != nil {
							t.Logf("---------------------- %s ----------------------", name)
							t.Logf("cannot read log: %v", err)
							t.Logf("---------------------- %s ----------------------", name)
						} else if len(log) > 0 {
							t.Logf("---------------------- %s ----------------------\n%s", name, string(log))
							t.Logf("---------------------- %s ----------------------", name)
						}
						f.Close()
					}
					if tsKeepFS {
						return nil
					}
					return os.RemoveAll(path)
				}

				break
			}
		}
	} else {
		stor = storage.NewMemStorage()
	}
	ts := &testStorage{
		t:       t,
		Storage: stor,
		closeFn: closeFn,
		opens:   make(map[uint64]bool),
	}
	ts.cond.L = &ts.mu
	return ts
}
Exemplo n.º 10
0
// NewMDServerLocal constructs a new MDServerLocal object that stores
// data in the directories specified as parameters to this function.
func NewMDServerLocal(config Config, handleDbfile string, mdDbfile string,
	branchDbfile string) (*MDServerLocal, error) {

	handleStorage, err := storage.OpenFile(handleDbfile)
	if err != nil {
		return nil, err
	}

	mdStorage, err := storage.OpenFile(mdDbfile)
	if err != nil {
		return nil, err
	}

	branchStorage, err := storage.OpenFile(branchDbfile)
	if err != nil {
		return nil, err
	}

	return newMDServerLocalWithStorage(config, handleStorage, mdStorage,
		branchStorage)
}
Exemplo n.º 11
0
// RecoverFile recovers and opens a DB with missing or corrupted manifest files
// for the given path. It will ignore any manifest files, valid or not.
// The DB must already exist or it will returns an error.
// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
//
// RecoverFile uses standard file-system backed storage implementation as desribed
// in the leveldb/storage package.
//
// The returned DB instance is goroutine-safe.
// The DB must be closed after use, by calling Close method.
func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
	stor, err := storage.OpenFile(path)
	if err != nil {
		return
	}
	db, err = Recover(stor, o)
	if err != nil {
		stor.Close()
	} else {
		db.closer = stor
	}
	return
}
Exemplo n.º 12
0
// OpenFile opens or creates a DB for the given path.
// The DB will be created if not exist, unless ErrorIfMissing is true.
// Also, if ErrorIfExist is true and the DB exist OpenFile will returns
// os.ErrExist error.
//
// OpenFile uses standard file-system backed storage implementation as
// desribed in the leveldb/storage package.
//
// OpenFile will return an error with type of ErrCorrupted if corruption
// detected in the DB. Corrupted DB can be recovered with Recover
// function.
//
// The returned DB instance is goroutine-safe.
// The DB must be closed after use, by calling Close method.
func OpenFile(path string, o *opt.Options) (db *DB, err error) {
	stor, err := storage.OpenFile(path, o.GetReadOnly())
	if err != nil {
		return
	}
	db, err = Open(stor, o)
	if err != nil {
		stor.Close()
	} else {
		db.closer = stor
	}
	return
}
Exemplo n.º 13
0
Arquivo: db.go Projeto: 29n/goleveldb
// OpenFile open or create database from given file.
//
// This is alias of:
//	stor, err := storage.OpenFile("path/to/db")
//	...
//	db, err := Open(stor, &opt.Options{})
//	...
func OpenFile(path string, o *opt.Options) (db *DB, err error) {
	stor, err := storage.OpenFile(path)
	if err != nil {
		return
	}
	db, err = Open(stor, o)
	if err == nil {
		db.closeCb = func() error {
			return stor.Close()
		}
	}
	return
}
Exemplo n.º 14
0
// RecoverDB recovers LevelDB database from corruption
func RecoverDB(path string) error {
	stor, err := storage.OpenFile(path, false)
	if err != nil {
		return err
	}

	db, err := leveldb.Recover(stor, nil)
	if err != nil {
		return err
	}

	db.Close()
	stor.Close()

	return nil
}
Exemplo n.º 15
0
func (ec *recoverCmd) Main() {
	ec.configuredCmd.Main()
	InitLog()
	path := openpgp.Config().Settings.TomlTree.Get("symflux.recon.leveldb.path").(string)
	stor, err := storage.OpenFile(path)
	if err != nil {
		die(err)
	}
	log.Println("database storage opened, recovering...")
	db, err := leveldb.Recover(stor, nil)
	if err != nil {
		die(err)
	}
	log.Println("recovery complete")
	db.Close()
}
Exemplo n.º 16
0
func NewMirageStorage(cfg *Config) *MirageStorage {
	fileStorage, err := storage.OpenFile(cfg.Storage.DataDir)
	if err != nil {
		fmt.Println("cannot open leveldb fileStorage")
		log.Fatal(err)
	}

	storage, err := leveldb.Open(fileStorage, &opt.Options{})
	if err != nil {
		fmt.Println("cannot open leveldb")
		log.Fatal(err)
	}

	ms := &MirageStorage{storage: storage}

	return ms
}
Exemplo n.º 17
0
func main() {
	flag.Parse()

	fmt.Printf("Using path: %s\n", filename)
	if child {
		fmt.Println("Child flag set.")
	}

	stor, err := storage.OpenFile(filename)
	if err != nil {
		fmt.Printf("Could not open storage: %s", err)
		os.Exit(10)
	}

	if !child {
		fmt.Println("Executing child -- first test (expecting error)")
		err := runChild()
		if err == nil {
			fmt.Println("Expecting error from child")
		} else if err.Error() != "exit status 10" {
			fmt.Println("Got unexpected error from child:", err)
		} else {
			fmt.Printf("Got error from child: %s (expected)\n", err)
		}
	}

	err = stor.Close()
	if err != nil {
		fmt.Printf("Error when closing storage: %s", err)
		os.Exit(11)
	}

	if !child {
		fmt.Println("Executing child -- second test")
		err := runChild()
		if err != nil {
			fmt.Println("Got unexpected error from child:", err)
		}
	}

	os.RemoveAll(filename)
}
Exemplo n.º 18
0
func NewStorage() *Storage {
	var (
		stor storage.Storage
		path string
	)
	if storageUseFS {
		for {
			storageMu.Lock()
			num := storageNum
			storageNum++
			storageMu.Unlock()
			path = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
			if _, err := os.Stat(path); os.IsNotExist(err) {
				stor, err = storage.OpenFile(path, false)
				ExpectWithOffset(1, err).NotTo(HaveOccurred(), "creating storage at %s", path)
				break
			}
		}
	} else {
		stor = storage.NewMemStorage()
	}
	s := &Storage{
		Storage: stor,
		path:    path,
		rand:    NewRand(),
		opens:   make(map[uint64]bool),
	}
	s.stallCond.L = &s.mu
	if s.path != "" {
		s.logI("using FS storage")
		s.logI("storage path: %s", s.path)
	} else {
		s.logI("using MEM storage")
	}
	return s
}
Exemplo n.º 19
0
func main() {
	flag.Parse()

	if enableBufferPool {
		bpool = util.NewBufferPool(opt.DefaultBlockSize + 128)
	}

	log.Printf("Test DB stored at %q", dbPath)
	if httpProf != "" {
		log.Printf("HTTP pprof listening at %q", httpProf)
		runtime.SetBlockProfileRate(1)
		go func() {
			if err := http.ListenAndServe(httpProf, nil); err != nil {
				log.Fatalf("HTTPPROF: %v", err)
			}
		}()
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	os.RemoveAll(dbPath)
	stor, err := storage.OpenFile(dbPath, false)
	if err != nil {
		log.Fatal(err)
	}
	tstor := &testingStorage{stor}
	defer tstor.Close()

	fatalf := func(err error, format string, v ...interface{}) {
		atomic.StoreUint32(&fail, 1)
		atomic.StoreUint32(&done, 1)
		log.Printf("FATAL: "+format, v...)
		if err != nil && errors.IsCorrupted(err) {
			cerr := err.(*errors.ErrCorrupted)
			if !cerr.Fd.Zero() && cerr.Fd.Type == storage.TypeTable {
				log.Print("FATAL: corruption detected, scanning...")
				if !tstor.scanTable(storage.FileDesc{Type: storage.TypeTable, Num: cerr.Fd.Num}, false) {
					log.Printf("FATAL: unable to find corrupted key/value pair in table %v", cerr.Fd)
				}
			}
		}
		runtime.Goexit()
	}

	if openFilesCacheCapacity == 0 {
		openFilesCacheCapacity = -1
	}
	o := &opt.Options{
		OpenFilesCacheCapacity: openFilesCacheCapacity,
		DisableBufferPool:      !enableBufferPool,
		DisableBlockCache:      !enableBlockCache,
		ErrorIfExist:           true,
		Compression:            opt.NoCompression,
	}
	if enableCompression {
		o.Compression = opt.DefaultCompression
	}

	db, err := leveldb.Open(tstor, o)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	var (
		mu              = &sync.Mutex{}
		gGetStat        = &latencyStats{}
		gIterStat       = &latencyStats{}
		gWriteStat      = &latencyStats{}
		gTrasactionStat = &latencyStats{}
		startTime       = time.Now()

		writeReq    = make(chan *leveldb.Batch)
		writeAck    = make(chan error)
		writeAckAck = make(chan struct{})
	)

	go func() {
		for b := range writeReq {

			var err error
			if mrand.Float64() < transactionProb {
				log.Print("> Write using transaction")
				gTrasactionStat.start()
				var tr *leveldb.Transaction
				if tr, err = db.OpenTransaction(); err == nil {
					if err = tr.Write(b, nil); err == nil {
						if err = tr.Commit(); err == nil {
							gTrasactionStat.record(b.Len())
						}
					} else {
						tr.Discard()
					}
				}
			} else {
				gWriteStat.start()
				if err = db.Write(b, nil); err == nil {
					gWriteStat.record(b.Len())
				}
			}
			writeAck <- err
			<-writeAckAck
		}
	}()

	go func() {
		for {
			time.Sleep(3 * time.Second)

			log.Print("------------------------")

			log.Printf("> Elapsed=%v", time.Now().Sub(startTime))
			mu.Lock()
			log.Printf("> GetLatencyMin=%v GetLatencyMax=%v GetLatencyAvg=%v GetRatePerSec=%d",
				gGetStat.min, gGetStat.max, gGetStat.avg(), gGetStat.ratePerSec())
			log.Printf("> IterLatencyMin=%v IterLatencyMax=%v IterLatencyAvg=%v IterRatePerSec=%d",
				gIterStat.min, gIterStat.max, gIterStat.avg(), gIterStat.ratePerSec())
			log.Printf("> WriteLatencyMin=%v WriteLatencyMax=%v WriteLatencyAvg=%v WriteRatePerSec=%d",
				gWriteStat.min, gWriteStat.max, gWriteStat.avg(), gWriteStat.ratePerSec())
			log.Printf("> TransactionLatencyMin=%v TransactionLatencyMax=%v TransactionLatencyAvg=%v TransactionRatePerSec=%d",
				gTrasactionStat.min, gTrasactionStat.max, gTrasactionStat.avg(), gTrasactionStat.ratePerSec())
			mu.Unlock()

			cachedblock, _ := db.GetProperty("leveldb.cachedblock")
			openedtables, _ := db.GetProperty("leveldb.openedtables")
			alivesnaps, _ := db.GetProperty("leveldb.alivesnaps")
			aliveiters, _ := db.GetProperty("leveldb.aliveiters")
			blockpool, _ := db.GetProperty("leveldb.blockpool")
			log.Printf("> BlockCache=%s OpenedTables=%s AliveSnaps=%s AliveIter=%s BlockPool=%q",
				cachedblock, openedtables, alivesnaps, aliveiters, blockpool)

			log.Print("------------------------")
		}
	}()

	for ns, numKey := range numKeys {
		func(ns, numKey int) {
			log.Printf("[%02d] STARTING: numKey=%d", ns, numKey)

			keys := make([][]byte, numKey)
			for i := range keys {
				keys[i] = randomData(nil, byte(ns), 1, uint32(i), keyLen)
			}

			wg.Add(1)
			go func() {
				var wi uint32
				defer func() {
					log.Printf("[%02d] WRITER DONE #%d", ns, wi)
					wg.Done()
				}()

				var (
					b       = new(leveldb.Batch)
					k2, v2  []byte
					nReader int32
				)
				for atomic.LoadUint32(&done) == 0 {
					log.Printf("[%02d] WRITER #%d", ns, wi)

					b.Reset()
					for _, k1 := range keys {
						k2 = randomData(k2, byte(ns), 2, wi, keyLen)
						v2 = randomData(v2, byte(ns), 3, wi, valueLen)
						b.Put(k2, v2)
						b.Put(k1, k2)
					}
					writeReq <- b
					if err := <-writeAck; err != nil {
						writeAckAck <- struct{}{}
						fatalf(err, "[%02d] WRITER #%d db.Write: %v", ns, wi, err)
					}

					snap, err := db.GetSnapshot()
					if err != nil {
						writeAckAck <- struct{}{}
						fatalf(err, "[%02d] WRITER #%d db.GetSnapshot: %v", ns, wi, err)
					}

					writeAckAck <- struct{}{}

					wg.Add(1)
					atomic.AddInt32(&nReader, 1)
					go func(snapwi uint32, snap *leveldb.Snapshot) {
						var (
							ri       int
							iterStat = &latencyStats{}
							getStat  = &latencyStats{}
						)
						defer func() {
							mu.Lock()
							gGetStat.add(getStat)
							gIterStat.add(iterStat)
							mu.Unlock()

							atomic.AddInt32(&nReader, -1)
							log.Printf("[%02d] READER #%d.%d DONE Snap=%v Alive=%d IterLatency=%v GetLatency=%v", ns, snapwi, ri, snap, atomic.LoadInt32(&nReader), iterStat.avg(), getStat.avg())
							snap.Release()
							wg.Done()
						}()

						stopi := snapwi + 3
						for (ri < 3 || atomic.LoadUint32(&wi) < stopi) && atomic.LoadUint32(&done) == 0 {
							var n int
							iter := snap.NewIterator(dataPrefixSlice(byte(ns), 1), nil)
							iterStat.start()
							for iter.Next() {
								k1 := iter.Key()
								k2 := iter.Value()
								iterStat.record(1)

								if dataNS(k2) != byte(ns) {
									fatalf(nil, "[%02d] READER #%d.%d K%d invalid in-key NS: want=%d got=%d", ns, snapwi, ri, n, ns, dataNS(k2))
								}

								kwritei := dataI(k2)
								if kwritei != snapwi {
									fatalf(nil, "[%02d] READER #%d.%d K%d invalid in-key iter num: %d", ns, snapwi, ri, n, kwritei)
								}

								getStat.start()
								v2, err := snap.Get(k2, nil)
								if err != nil {
									fatalf(err, "[%02d] READER #%d.%d K%d snap.Get: %v\nk1: %x\n -> k2: %x", ns, snapwi, ri, n, err, k1, k2)
								}
								getStat.record(1)

								if checksum0, checksum1 := dataChecksum(v2); checksum0 != checksum1 {
									err := &errors.ErrCorrupted{Fd: storage.FileDesc{0xff, 0}, Err: fmt.Errorf("v2: %x: checksum mismatch: %v vs %v", v2, checksum0, checksum1)}
									fatalf(err, "[%02d] READER #%d.%d K%d snap.Get: %v\nk1: %x\n -> k2: %x", ns, snapwi, ri, n, err, k1, k2)
								}

								n++
								iterStat.start()
							}
							iter.Release()
							if err := iter.Error(); err != nil {
								fatalf(err, "[%02d] READER #%d.%d K%d iter.Error: %v", ns, snapwi, ri, numKey, err)
							}
							if n != numKey {
								fatalf(nil, "[%02d] READER #%d.%d missing keys: want=%d got=%d", ns, snapwi, ri, numKey, n)
							}

							ri++
						}
					}(wi, snap)

					atomic.AddUint32(&wi, 1)
				}
			}()

			delB := new(leveldb.Batch)
			wg.Add(1)
			go func() {
				var (
					i        int
					iterStat = &latencyStats{}
				)
				defer func() {
					log.Printf("[%02d] SCANNER DONE #%d", ns, i)
					wg.Done()
				}()

				time.Sleep(2 * time.Second)

				for atomic.LoadUint32(&done) == 0 {
					var n int
					delB.Reset()
					iter := db.NewIterator(dataNsSlice(byte(ns)), nil)
					iterStat.start()
					for iter.Next() && atomic.LoadUint32(&done) == 0 {
						k := iter.Key()
						v := iter.Value()
						iterStat.record(1)

						for ci, x := range [...][]byte{k, v} {
							checksum0, checksum1 := dataChecksum(x)
							if checksum0 != checksum1 {
								if ci == 0 {
									fatalf(nil, "[%02d] SCANNER %d.%d invalid key checksum: want %d, got %d\n%x -> %x", ns, i, n, checksum0, checksum1, k, v)
								} else {
									fatalf(nil, "[%02d] SCANNER %d.%d invalid value checksum: want %d, got %d\n%x -> %x", ns, i, n, checksum0, checksum1, k, v)
								}
							}
						}

						if dataPrefix(k) == 2 || mrand.Int()%999 == 0 {
							delB.Delete(k)
						}

						n++
						iterStat.start()
					}
					iter.Release()
					if err := iter.Error(); err != nil {
						fatalf(err, "[%02d] SCANNER #%d.%d iter.Error: %v", ns, i, n, err)
					}

					if n > 0 {
						log.Printf("[%02d] SCANNER #%d IterLatency=%v", ns, i, iterStat.avg())
					}

					if delB.Len() > 0 && atomic.LoadUint32(&done) == 0 {
						t := time.Now()
						writeReq <- delB
						if err := <-writeAck; err != nil {
							writeAckAck <- struct{}{}
							fatalf(err, "[%02d] SCANNER #%d db.Write: %v", ns, i, err)
						} else {
							writeAckAck <- struct{}{}
						}
						log.Printf("[%02d] SCANNER #%d Deleted=%d Time=%v", ns, i, delB.Len(), time.Now().Sub(t))
					}

					i++
				}
			}()
		}(ns, numKey)
	}

	go func() {
		sig := make(chan os.Signal)
		signal.Notify(sig, os.Interrupt, os.Kill)
		log.Printf("Got signal: %v, exiting...", <-sig)
		atomic.StoreUint32(&done, 1)
	}()

	wg.Wait()
}