func (db *DB) Destroy() {
db.Close()
opts := levigo.NewOptions()
defer opts.Close()
levigo.DestroyDatabase(db.cfg.Path, opts)
}
func (buckets *Database) DestroyBucket(name string) error {
if _, ok := buckets.DBMap[name]; ok {
delete(buckets.DBMap, name)
return levigo.DestroyDatabase(name, levigo.NewOptions())
}
return nil
}
// please don't change this function.
func (sm *ShardMaster) Kill() {
// Kill the server
DPrintfPersist("\n%v: Killing the server", sm.me)
sm.dead = true
if sm.l != nil {
sm.l.Close()
}
sm.px.Kill()
// Close the database
if sm.persistent && !sm.dbClosed {
sm.dbLock.Lock()
sm.db.Close()
sm.dbReadOptions.Close()
sm.dbWriteOptions.Close()
sm.dbLock.Unlock()
sm.dbClosed = true
}
// Destroy the database
if sm.persistent && !sm.dbDeleted {
DPrintfPersist("\n%v: Destroying database... ", sm.me)
err := levigo.DestroyDatabase(sm.dbName, sm.dbOpts)
if err != nil {
DPrintfPersist("\terror")
} else {
DPrintfPersist("\tsuccess")
sm.dbDeleted = true
}
}
}
func main() {
dbname := "leveldb"
opts := levigo.NewOptions()
opts.SetCache(levigo.NewLRUCache(3 << 20))
opts.SetCreateIfMissing(true)
_ = levigo.DestroyDatabase(dbname, opts)
db, _ := levigo.Open(dbname, opts)
wo := levigo.NewWriteOptions()
ro := levigo.NewReadOptions()
start := time.Now()
for i := 0; i < 10e4; i++ {
db.Put(wo, []byte(fmt.Sprintf("a%v", i)), []byte(strconv.Itoa(i)))
}
for i := 0; i < 10e4; i++ {
db.Get(ro, []byte(fmt.Sprintf("a%v", i)))
}
for i := 0; i < 10e4; i++ {
db.Delete(wo, []byte(fmt.Sprintf("a%v", i)))
}
duration := time.Since(start)
log.Printf("Elapsed: %v.", duration)
}
func TestTrigger(t *testing.T) {
opts := levigo.NewOptions()
levigo.DestroyDatabase("test.ldb", opts)
// opts.SetCache(levigo.NewLRUCache(3<<30))
opts.SetCreateIfMissing(true)
db, err := levigo.Open("test.ldb", opts)
if err != nil {
t.Fatal(err)
}
ro := levigo.NewReadOptions()
wo := levigo.NewWriteOptions()
sub1 := sublevel.Sublevel(db, "input")
index := sublevel.Sublevel(db, "index")
job := sublevel.Sublevel(db, "job")
task := Trigger(sub1, job, func(key, value []byte) []byte {
if strings.HasPrefix(string(key), "Doc_") {
return key
}
return nil
}, func(key, value []byte) bool {
doc := make(map[string]string)
err := json.Unmarshal(value, &doc)
if err != nil {
t.Fatal(err)
}
index.Put(wo, []byte(doc["id"]), []byte(doc["number"]))
// Make sure that the next task invocation comes in concurrently to this one
time.Sleep(300 * time.Millisecond)
return true
})
sub1.Put(wo, []byte("foobar"), []byte("do-not-process"))
// Two put operations which will both trigger the task for the same taskKey.
sub1.Put(wo, []byte("Doc_1"), []byte("{\"id\":\"01234\", \"number\": \"42\"}"))
sub1.Put(wo, []byte("Doc_1"), []byte("{\"id\":\"01234\", \"number\": \"43\"}"))
val, err := sub1.Get(ro, []byte("Doc_1"))
if err != nil || string(val) != "{\"id\":\"01234\", \"number\": \"43\"}" {
t.Fatal(err, string(val))
}
time.Sleep(800 * time.Millisecond)
val, err = index.Get(ro, []byte("01234"))
if err != nil || string(val) != "43" {
t.Fatal(err, string(val))
}
task.Close()
ro.Close()
wo.Close()
db.Close()
}
func (engine *LevelDbEngine) Destroy() error {
engine.db.Close()
err := levigo.DestroyDatabase(engine.config.DataPath, engine.dbOptions)
if err != nil {
return err
}
engine.dbOptions.Close()
return nil
}
//
// tell the peer to shut itself down.
// deletes disk contents
//
func (px *Paxos) Kill() {
// Kill the server
px.KillSaveDisk()
// Destroy the database
if px.persistent && !px.dbDeleted {
DPrintfPersist("\n%v (L%v): Destroying database... ", px.me)
err := levigo.DestroyDatabase(px.dbName, px.dbOpts)
if err != nil {
DPrintfPersist("\terror")
} else {
DPrintfPersist("\tsuccess")
px.dbDeleted = true
}
}
}
func TestMap(t *testing.T) {
opts := levigo.NewOptions()
levigo.DestroyDatabase("test.ldb", opts)
// opts.SetCache(levigo.NewLRUCache(3<<30))
opts.SetCreateIfMissing(true)
db, err := levigo.Open("test.ldb", opts)
if err != nil {
t.Fatal(err)
}
ro := levigo.NewReadOptions()
wo := levigo.NewWriteOptions()
sub1 := sublevel.Sublevel(db, "input")
index := sublevel.Sublevel(db, "index")
mapcount := 0
task := Map(sub1, index, "job", func(key, value []byte, emit EmitFunc) {
mapcount++
ingredients := strings.Split(string(value), ",")
for _, ingredient := range ingredients {
// println("emit", ingredient, string(key))
emit([]byte(ingredient), key)
}
})
sub1.Put(wo, []byte("Sushi"), []byte("Fish,Rice"))
sub1.Put(wo, []byte("Forelle Blau"), []byte("Fish,Potatoes"))
sub1.Put(wo, []byte("Wiener Schnitzel"), []byte("Fish,Potatoes"))
sub1.Put(wo, []byte("Pulled Pork"), []byte("Pig,ColeSlaw"))
// Wait until mapping has completed
time.Sleep(500 * time.Millisecond)
if mapcount != 4 {
t.Fatal(mapcount)
}
correct1 := []string{"Sushi", "Forelle Blau", "Wiener Schnitzel"}
it := task.NewIterator("Fish")
i := 0
for it.SeekToFirst(); it.Valid(); it.Next() {
if string(it.Value()) != correct1[i] {
t.Fatal(it.Value())
}
i++
}
if i != len(correct1) {
t.Fatal(i)
}
sub1.Put(wo, []byte("Wiener Schnitzel"), []byte("Pig,Potatoes"))
// Wait until mapping has completed
time.Sleep(500 * time.Millisecond)
if mapcount != 5 {
t.Fatal(mapcount)
}
correct2 := []string{"Sushi", "Forelle Blau"}
i = 0
it = task.NewIterator("Fish")
for it.SeekToFirst(); it.Valid(); it.Next() {
if string(it.Value()) != correct2[i] {
t.Fatal(it.Value())
}
i++
}
if i != len(correct2) {
t.Fatal(i)
}
correct3 := []string{"Sushi"}
sub1.Delete(wo, []byte("Forelle Blau"))
// Wait until mapping has completed
time.Sleep(500 * time.Millisecond)
if mapcount != 5 {
t.Fatal(mapcount)
}
it = task.NewIterator("Fish")
i = 0
for it.SeekToFirst(); it.Valid(); it.Next() {
if string(it.Value()) != correct3[i] {
t.Fatal(it.Value())
}
i++
}
if i != len(correct3) {
t.Fatal(i)
}
/*
it2 := db.NewIterator(ro)
for it2.SeekToFirst(); it2.Valid(); it2.Next() {
println("DB", len(it2.Key()), string(it2.Key()), string(it2.Value()))
}
*/
task.Close()
ro.Close()
wo.Close()
db.Close()
}
func (ulevel) DestroyDatabase(name string, o level.UnderlyingOptions) error {
return levigo.DestroyDatabase(name, o.(opts).Options)
}
func TestReduce(t *testing.T) {
opts := levigo.NewOptions()
levigo.DestroyDatabase("test.ldb", opts)
// opts.SetCache(levigo.NewLRUCache(3<<30))
opts.SetCreateIfMissing(true)
db, err := levigo.Open("test.ldb", opts)
if err != nil {
t.Fatal(err)
}
ro := levigo.NewReadOptions()
wo := levigo.NewWriteOptions()
sub1 := sublevel.Sublevel(db, "input")
index := sublevel.Sublevel(db, "index")
reduce := sublevel.Sublevel(db, "reduce")
mapcount := 0
task := Map(sub1, index, "mapjob", func(key, value []byte, emit EmitFunc) {
mapcount++
doc := make(map[string]string)
err := json.Unmarshal(value, &doc)
if err != nil {
t.Fatal(err)
}
emit([]string{doc["Country"], doc["State"], doc["City"]}, doc["Kind"])
})
reducecount := 0
rereducecount := 0
task2 := Reduce(index, reduce, "mapjob2", func(acc interface{}, value []byte) interface{} {
reducecount++
var kind string
err := json.Unmarshal(value, &kind)
if err != nil {
t.Fatal(err)
}
val := acc.(map[string]int)
if n, ok := val[kind]; ok {
val[kind] = n + 1
} else {
val[kind] = 1
}
return val
}, func(acc interface{}, value []byte) interface{} {
rereducecount++
var acc2 map[string]int
err := json.Unmarshal(value, &acc2)
if err != nil {
t.Fatal(err)
}
val := acc.(map[string]int)
for k, v := range acc2 {
if n, ok := val[k]; ok {
val[k] = n + v
} else {
val[k] = v
}
}
return val
}, func() interface{} {
return make(map[string]int)
}, 0)
sub1.Put(wo, []byte("Bella Vista"), []byte("{\"Country\":\"Germany\",\"State\":\"NRW\",\"City\":\"Duisburg\",\"Kind\":\"Pizza\"}"))
sub1.Put(wo, []byte("Tokio"), []byte("{\"Country\":\"Germany\",\"State\":\"NRW\",\"City\":\"Düsseldorf\",\"Kind\":\"Sushi\"}"))
sub1.Put(wo, []byte("Maria"), []byte("{\"Country\":\"Germany\",\"State\":\"NRW\",\"City\":\"Duisburg\",\"Kind\":\"Pizza\"}"))
sub1.Put(wo, []byte("Formagio"), []byte("{\"Country\":\"Germany\",\"State\":\"NRW\",\"City\":\"Essen\",\"Kind\":\"Pizza\"}"))
sub1.Put(wo, []byte("Fungi"), []byte("{\"Country\":\"Germany\",\"State\":\"Hessen\",\"City\":\"Frankfurt\",\"Kind\":\"Pizza\"}"))
sub1.Put(wo, []byte("New Tokio"), []byte("{\"Country\":\"Germany\",\"State\":\"Bayern\",\"City\":\"Munich\",\"Kind\":\"Sushi\"}"))
sub1.Put(wo, []byte("San Torino"), []byte("{\"Country\":\"USA\",\"State\":\"CA\",\"City\":\"San Jose\",\"Kind\":\"Pizza\"}"))
sub1.Delete(wo, []byte("Formagio"))
// Wait until map reduce has completed
time.Sleep(1000 * time.Millisecond)
it2 := db.NewIterator(ro)
for it2.SeekToFirst(); it2.Valid(); it2.Next() {
println("DB", len(it2.Key()), string(it2.Key()), string(it2.Value()))
}
task.Close()
task2.Close()
ro.Close()
wo.Close()
db.Close()
}
func TestSublevel(t *testing.T) {
opts := levigo.NewOptions()
levigo.DestroyDatabase("test.ldb", opts)
// opts.SetCache(levigo.NewLRUCache(3<<30))
opts.SetCreateIfMissing(true)
db, err := levigo.Open("test.ldb", opts)
if err != nil {
t.Fatal(err)
}
ro := levigo.NewReadOptions()
wo := levigo.NewWriteOptions()
sub1 := Sublevel(db, "foo")
sub2 := Sublevel(db, "bar")
err = sub1.Put(wo, []byte("hudel0000"), []byte("Test1_0000"))
if err != nil {
t.Fatal(err)
}
err = sub2.Put(wo, []byte("hudel0000"), []byte("Test2_0000"))
if err != nil {
t.Fatal(err)
}
val1, err := sub1.Get(ro, []byte("hudel0000"))
if err != nil || string(val1) != "Test1_0000" {
t.Fatal(err, string(val1))
}
val2, err := sub2.Get(ro, []byte("hudel0000"))
if err != nil || string(val2) != "Test2_0000" {
t.Fatal(err)
}
for i := 1; i < 100; i++ {
err = sub1.Put(wo, []byte("hudel"+fmt.Sprintf("%04d", i)), []byte("Test1_"+fmt.Sprintf("%04d", i)))
if err != nil {
t.Fatal(err)
}
}
for i := 1; i < 100; i++ {
err = sub2.Put(wo, []byte("hudel"+fmt.Sprintf("%04d", i)), []byte("Test2_"+fmt.Sprintf("%04d", i)))
if err != nil {
t.Fatal(err)
}
}
count := 0
it := sub1.NewIterator(ro)
for it.SeekToFirst(); it.Valid(); it.Next() {
if "hudel"+fmt.Sprintf("%04d", count) != string(it.Key()) {
t.Fatal(string(it.Key()))
}
if "Test1_"+fmt.Sprintf("%04d", count) != string(it.Value()) {
t.Fatal(string(it.Value()))
}
val, err := sub1.Get(ro, []byte("hudel"+fmt.Sprintf("%04d", count)))
if err != nil || string(val) != "Test1_"+fmt.Sprintf("%04d", count) {
t.Fatal(err, count, string(val))
}
count++
}
if count != 100 {
t.Fatalf("Iterator stopped too early or too late %v", count)
}
count = 99
for it.SeekToLast(); it.Valid(); it.Prev() {
if "hudel"+fmt.Sprintf("%04d", count) != string(it.Key()) {
t.Fatal(string(it.Key()))
}
if "Test1_"+fmt.Sprintf("%04d", count) != string(it.Value()) {
t.Fatal(string(it.Value()))
}
val, err := sub1.Get(ro, []byte("hudel"+fmt.Sprintf("%04d", count)))
if err != nil || string(val) != "Test1_"+fmt.Sprintf("%04d", count) {
t.Fatal(err, count, string(val))
}
count--
}
if count != -1 {
t.Fatalf("Iterator stopped too early or too late %v", count)
}
it.Close()
count = 0
it = sub2.NewIterator(ro)
for it.SeekToFirst(); it.Valid(); it.Next() {
val, err := sub2.Get(ro, []byte("hudel"+fmt.Sprintf("%04d", count)))
if err != nil || string(val) != "Test2_"+fmt.Sprintf("%04d", count) {
t.Fatal(err)
}
count++
}
if count != 100 {
t.Fatalf("Iterator stopped too early %v", count)
}
it.Close()
ro.Close()
wo.Close()
db.Close()
}
func TestHook(t *testing.T) {
opts := levigo.NewOptions()
levigo.DestroyDatabase("test.ldb", opts)
// opts.SetCache(levigo.NewLRUCache(3<<30))
opts.SetCreateIfMissing(true)
db, err := levigo.Open("test.ldb", opts)
if err != nil {
t.Fatal(err)
}
ro := levigo.NewReadOptions()
wo := levigo.NewWriteOptions()
sub1 := Sublevel(db, "input")
sub2 := Sublevel(db, "output")
sub3 := Sublevel(db, "output2")
sub1.Pre(func(key, value []byte, hook *Hook) {
if value != nil {
hook.Put(append([]byte("D_"), key...), value, sub2)
} else {
hook.Delete(append([]byte("D_"), key...), sub2)
}
})
sub2.Pre(func(key, value []byte, hook *Hook) {
if value != nil {
hook.Put(append([]byte("X_"), key...), value, sub3)
} else {
hook.Delete(append([]byte("X_"), key...), sub3)
}
})
post1Called := 0
post3Called := 0
sub1.Post(func(key, value []byte) {
post1Called++
val, err := sub1.Get(ro, key)
if err != nil || bytes.Compare(val, value) != 0 {
t.Fatal(err, string(val), string(value))
}
})
sub3.Post(func(key, value []byte) {
post3Called++
val, err := sub3.Get(ro, key)
if err != nil || bytes.Compare(val, value) != 0 {
t.Fatal(err, "read=", string(val), "hook=", string(value))
}
})
sub1.Put(wo, []byte("Name"), []byte("Horst"))
val, err := sub1.Get(ro, []byte("Name"))
if err != nil || string(val) != "Horst" {
t.Fatal(err, string(val))
}
val, err = sub2.Get(ro, []byte("D_Name"))
if err != nil || string(val) != "Horst" {
t.Fatal(err, string(val))
}
val, err = sub3.Get(ro, []byte("X_D_Name"))
if err != nil || string(val) != "Horst" {
t.Fatal(err, string(val))
}
sub1.Delete(wo, []byte("Name"))
val, err = sub1.Get(ro, []byte("Name"))
if err != nil || val != nil {
t.Fatal(err, string(val))
}
val, err = sub2.Get(ro, []byte("D_Name"))
if err != nil || val != nil {
t.Fatal(err, string(val))
}
val, err = sub3.Get(ro, []byte("X_D_Name"))
if err != nil || val != nil {
t.Fatal(err, string(val))
}
sub1.Put(wo, []byte("Color"), []byte("Red"))
batch := sub1.NewWriteBatch()
batch.Put([]byte("Sound"), []byte("Ping"))
batch.Delete([]byte("Color"))
sub1.Write(wo, batch)
if post1Called != 5 {
t.Fatal(post1Called)
}
if post3Called != 5 {
t.Fatal(post3Called)
}
val, err = sub1.Get(ro, []byte("Color"))
if err != nil || val != nil {
t.Fatal(err, string(val))
}
val, err = sub2.Get(ro, []byte("D_Color"))
if err != nil || val != nil {
t.Fatal(err, string(val))
}
val, err = sub3.Get(ro, []byte("X_D_Color"))
if err != nil || val != nil {
t.Fatal(err, string(val))
}
val, err = sub1.Get(ro, []byte("Sound"))
if err != nil || string(val) != "Ping" {
t.Fatal(err, string(val))
}
val, err = sub2.Get(ro, []byte("D_Sound"))
if err != nil || string(val) != "Ping" {
t.Fatal(err, string(val))
}
val, err = sub3.Get(ro, []byte("X_D_Sound"))
if err != nil || string(val) != "Ping" {
t.Fatal(err, string(val))
}
ro.Close()
wo.Close()
db.Close()
}