Exemple #1
0
func (db *DB) Destroy() {
	db.Close()
	opts := levigo.NewOptions()
	defer opts.Close()

	levigo.DestroyDatabase(db.cfg.Path, opts)
}
Exemple #2
0
func (buckets *Database) DestroyBucket(name string) error {
	if _, ok := buckets.DBMap[name]; ok {
		delete(buckets.DBMap, name)
		return levigo.DestroyDatabase(name, levigo.NewOptions())
	}
	return nil
}
Exemple #3
0
// please don't change this function.
func (sm *ShardMaster) Kill() {
	// Kill the server
	DPrintfPersist("\n%v: Killing the server", sm.me)
	sm.dead = true
	if sm.l != nil {
		sm.l.Close()
	}
	sm.px.Kill()

	// Close the database
	if sm.persistent && !sm.dbClosed {
		sm.dbLock.Lock()
		sm.db.Close()
		sm.dbReadOptions.Close()
		sm.dbWriteOptions.Close()
		sm.dbLock.Unlock()
		sm.dbClosed = true
	}

	// Destroy the database
	if sm.persistent && !sm.dbDeleted {
		DPrintfPersist("\n%v: Destroying database... ", sm.me)
		err := levigo.DestroyDatabase(sm.dbName, sm.dbOpts)
		if err != nil {
			DPrintfPersist("\terror")
		} else {
			DPrintfPersist("\tsuccess")
			sm.dbDeleted = true
		}
	}
}
Exemple #4
0
func main() {
	dbname := "leveldb"
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(3 << 20))
	opts.SetCreateIfMissing(true)
	_ = levigo.DestroyDatabase(dbname, opts)
	db, _ := levigo.Open(dbname, opts)

	wo := levigo.NewWriteOptions()
	ro := levigo.NewReadOptions()

	start := time.Now()

	for i := 0; i < 10e4; i++ {
		db.Put(wo, []byte(fmt.Sprintf("a%v", i)), []byte(strconv.Itoa(i)))
	}
	for i := 0; i < 10e4; i++ {
		db.Get(ro, []byte(fmt.Sprintf("a%v", i)))
	}
	for i := 0; i < 10e4; i++ {
		db.Delete(wo, []byte(fmt.Sprintf("a%v", i)))
	}
	duration := time.Since(start)
	log.Printf("Elapsed: %v.", duration)
}
Exemple #5
0
func TestTrigger(t *testing.T) {
	opts := levigo.NewOptions()
	levigo.DestroyDatabase("test.ldb", opts)
	// opts.SetCache(levigo.NewLRUCache(3<<30))
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open("test.ldb", opts)
	if err != nil {
		t.Fatal(err)
	}

	ro := levigo.NewReadOptions()
	wo := levigo.NewWriteOptions()

	sub1 := sublevel.Sublevel(db, "input")
	index := sublevel.Sublevel(db, "index")
	job := sublevel.Sublevel(db, "job")

	task := Trigger(sub1, job, func(key, value []byte) []byte {
		if strings.HasPrefix(string(key), "Doc_") {
			return key
		}
		return nil
	}, func(key, value []byte) bool {
		doc := make(map[string]string)
		err := json.Unmarshal(value, &doc)
		if err != nil {
			t.Fatal(err)
		}
		index.Put(wo, []byte(doc["id"]), []byte(doc["number"]))
		// Make sure that the next task invocation comes in concurrently to this one
		time.Sleep(300 * time.Millisecond)
		return true
	})

	sub1.Put(wo, []byte("foobar"), []byte("do-not-process"))

	// Two put operations which will both trigger the task for the same taskKey.
	sub1.Put(wo, []byte("Doc_1"), []byte("{\"id\":\"01234\", \"number\": \"42\"}"))
	sub1.Put(wo, []byte("Doc_1"), []byte("{\"id\":\"01234\", \"number\": \"43\"}"))

	val, err := sub1.Get(ro, []byte("Doc_1"))
	if err != nil || string(val) != "{\"id\":\"01234\", \"number\": \"43\"}" {
		t.Fatal(err, string(val))
	}

	time.Sleep(800 * time.Millisecond)

	val, err = index.Get(ro, []byte("01234"))
	if err != nil || string(val) != "43" {
		t.Fatal(err, string(val))
	}

	task.Close()

	ro.Close()
	wo.Close()
	db.Close()
}
Exemple #6
0
func (engine *LevelDbEngine) Destroy() error {
	engine.db.Close()
	err := levigo.DestroyDatabase(engine.config.DataPath, engine.dbOptions)
	if err != nil {
		return err
	}
	engine.dbOptions.Close()
	return nil
}
Exemple #7
0
//
// tell the peer to shut itself down.
// deletes disk contents
//
func (px *Paxos) Kill() {
	// Kill the server
	px.KillSaveDisk()

	// Destroy the database
	if px.persistent && !px.dbDeleted {
		DPrintfPersist("\n%v (L%v): Destroying database... ", px.me)
		err := levigo.DestroyDatabase(px.dbName, px.dbOpts)
		if err != nil {
			DPrintfPersist("\terror")
		} else {
			DPrintfPersist("\tsuccess")
			px.dbDeleted = true
		}
	}
}
Exemple #8
0
func TestMap(t *testing.T) {
	opts := levigo.NewOptions()
	levigo.DestroyDatabase("test.ldb", opts)
	// opts.SetCache(levigo.NewLRUCache(3<<30))
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open("test.ldb", opts)
	if err != nil {
		t.Fatal(err)
	}

	ro := levigo.NewReadOptions()
	wo := levigo.NewWriteOptions()

	sub1 := sublevel.Sublevel(db, "input")
	index := sublevel.Sublevel(db, "index")

	mapcount := 0
	task := Map(sub1, index, "job", func(key, value []byte, emit EmitFunc) {
		mapcount++
		ingredients := strings.Split(string(value), ",")
		for _, ingredient := range ingredients {
			//			println("emit", ingredient, string(key))
			emit([]byte(ingredient), key)
		}
	})

	sub1.Put(wo, []byte("Sushi"), []byte("Fish,Rice"))
	sub1.Put(wo, []byte("Forelle Blau"), []byte("Fish,Potatoes"))
	sub1.Put(wo, []byte("Wiener Schnitzel"), []byte("Fish,Potatoes"))
	sub1.Put(wo, []byte("Pulled Pork"), []byte("Pig,ColeSlaw"))

	// Wait until mapping has completed
	time.Sleep(500 * time.Millisecond)

	if mapcount != 4 {
		t.Fatal(mapcount)
	}

	correct1 := []string{"Sushi", "Forelle Blau", "Wiener Schnitzel"}
	it := task.NewIterator("Fish")
	i := 0
	for it.SeekToFirst(); it.Valid(); it.Next() {
		if string(it.Value()) != correct1[i] {
			t.Fatal(it.Value())
		}
		i++
	}
	if i != len(correct1) {
		t.Fatal(i)
	}

	sub1.Put(wo, []byte("Wiener Schnitzel"), []byte("Pig,Potatoes"))

	// Wait until mapping has completed
	time.Sleep(500 * time.Millisecond)

	if mapcount != 5 {
		t.Fatal(mapcount)
	}

	correct2 := []string{"Sushi", "Forelle Blau"}
	i = 0
	it = task.NewIterator("Fish")
	for it.SeekToFirst(); it.Valid(); it.Next() {
		if string(it.Value()) != correct2[i] {
			t.Fatal(it.Value())
		}
		i++
	}
	if i != len(correct2) {
		t.Fatal(i)
	}

	correct3 := []string{"Sushi"}
	sub1.Delete(wo, []byte("Forelle Blau"))

	// Wait until mapping has completed
	time.Sleep(500 * time.Millisecond)

	if mapcount != 5 {
		t.Fatal(mapcount)
	}

	it = task.NewIterator("Fish")
	i = 0
	for it.SeekToFirst(); it.Valid(); it.Next() {
		if string(it.Value()) != correct3[i] {
			t.Fatal(it.Value())
		}
		i++
	}
	if i != len(correct3) {
		t.Fatal(i)
	}

	/*
		it2 := db.NewIterator(ro)
		for it2.SeekToFirst(); it2.Valid(); it2.Next() {
			println("DB", len(it2.Key()), string(it2.Key()), string(it2.Value()))
		}
	*/

	task.Close()

	ro.Close()
	wo.Close()
	db.Close()
}
Exemple #9
0
func (ulevel) DestroyDatabase(name string, o level.UnderlyingOptions) error {
	return levigo.DestroyDatabase(name, o.(opts).Options)
}
Exemple #10
0
func TestReduce(t *testing.T) {
	opts := levigo.NewOptions()
	levigo.DestroyDatabase("test.ldb", opts)
	// opts.SetCache(levigo.NewLRUCache(3<<30))
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open("test.ldb", opts)
	if err != nil {
		t.Fatal(err)
	}

	ro := levigo.NewReadOptions()
	wo := levigo.NewWriteOptions()

	sub1 := sublevel.Sublevel(db, "input")
	index := sublevel.Sublevel(db, "index")
	reduce := sublevel.Sublevel(db, "reduce")

	mapcount := 0
	task := Map(sub1, index, "mapjob", func(key, value []byte, emit EmitFunc) {
		mapcount++
		doc := make(map[string]string)
		err := json.Unmarshal(value, &doc)
		if err != nil {
			t.Fatal(err)
		}
		emit([]string{doc["Country"], doc["State"], doc["City"]}, doc["Kind"])
	})

	reducecount := 0
	rereducecount := 0
	task2 := Reduce(index, reduce, "mapjob2", func(acc interface{}, value []byte) interface{} {
		reducecount++
		var kind string
		err := json.Unmarshal(value, &kind)
		if err != nil {
			t.Fatal(err)
		}
		val := acc.(map[string]int)
		if n, ok := val[kind]; ok {
			val[kind] = n + 1
		} else {
			val[kind] = 1
		}
		return val
	}, func(acc interface{}, value []byte) interface{} {
		rereducecount++
		var acc2 map[string]int
		err := json.Unmarshal(value, &acc2)
		if err != nil {
			t.Fatal(err)
		}
		val := acc.(map[string]int)
		for k, v := range acc2 {
			if n, ok := val[k]; ok {
				val[k] = n + v
			} else {
				val[k] = v
			}
		}
		return val
	}, func() interface{} {
		return make(map[string]int)
	}, 0)

	sub1.Put(wo, []byte("Bella Vista"), []byte("{\"Country\":\"Germany\",\"State\":\"NRW\",\"City\":\"Duisburg\",\"Kind\":\"Pizza\"}"))
	sub1.Put(wo, []byte("Tokio"), []byte("{\"Country\":\"Germany\",\"State\":\"NRW\",\"City\":\"Düsseldorf\",\"Kind\":\"Sushi\"}"))
	sub1.Put(wo, []byte("Maria"), []byte("{\"Country\":\"Germany\",\"State\":\"NRW\",\"City\":\"Duisburg\",\"Kind\":\"Pizza\"}"))
	sub1.Put(wo, []byte("Formagio"), []byte("{\"Country\":\"Germany\",\"State\":\"NRW\",\"City\":\"Essen\",\"Kind\":\"Pizza\"}"))
	sub1.Put(wo, []byte("Fungi"), []byte("{\"Country\":\"Germany\",\"State\":\"Hessen\",\"City\":\"Frankfurt\",\"Kind\":\"Pizza\"}"))
	sub1.Put(wo, []byte("New Tokio"), []byte("{\"Country\":\"Germany\",\"State\":\"Bayern\",\"City\":\"Munich\",\"Kind\":\"Sushi\"}"))
	sub1.Put(wo, []byte("San Torino"), []byte("{\"Country\":\"USA\",\"State\":\"CA\",\"City\":\"San Jose\",\"Kind\":\"Pizza\"}"))
	sub1.Delete(wo, []byte("Formagio"))

	// Wait until map reduce has completed
	time.Sleep(1000 * time.Millisecond)

	it2 := db.NewIterator(ro)
	for it2.SeekToFirst(); it2.Valid(); it2.Next() {
		println("DB", len(it2.Key()), string(it2.Key()), string(it2.Value()))
	}

	task.Close()
	task2.Close()

	ro.Close()
	wo.Close()
	db.Close()
}
Exemple #11
0
func TestSublevel(t *testing.T) {
	opts := levigo.NewOptions()
	levigo.DestroyDatabase("test.ldb", opts)
	// opts.SetCache(levigo.NewLRUCache(3<<30))
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open("test.ldb", opts)
	if err != nil {
		t.Fatal(err)
	}

	ro := levigo.NewReadOptions()
	wo := levigo.NewWriteOptions()

	sub1 := Sublevel(db, "foo")
	sub2 := Sublevel(db, "bar")

	err = sub1.Put(wo, []byte("hudel0000"), []byte("Test1_0000"))
	if err != nil {
		t.Fatal(err)
	}
	err = sub2.Put(wo, []byte("hudel0000"), []byte("Test2_0000"))
	if err != nil {
		t.Fatal(err)
	}

	val1, err := sub1.Get(ro, []byte("hudel0000"))
	if err != nil || string(val1) != "Test1_0000" {
		t.Fatal(err, string(val1))
	}
	val2, err := sub2.Get(ro, []byte("hudel0000"))
	if err != nil || string(val2) != "Test2_0000" {
		t.Fatal(err)
	}
	for i := 1; i < 100; i++ {
		err = sub1.Put(wo, []byte("hudel"+fmt.Sprintf("%04d", i)), []byte("Test1_"+fmt.Sprintf("%04d", i)))
		if err != nil {
			t.Fatal(err)
		}
	}

	for i := 1; i < 100; i++ {
		err = sub2.Put(wo, []byte("hudel"+fmt.Sprintf("%04d", i)), []byte("Test2_"+fmt.Sprintf("%04d", i)))
		if err != nil {
			t.Fatal(err)
		}
	}

	count := 0
	it := sub1.NewIterator(ro)
	for it.SeekToFirst(); it.Valid(); it.Next() {
		if "hudel"+fmt.Sprintf("%04d", count) != string(it.Key()) {
			t.Fatal(string(it.Key()))
		}
		if "Test1_"+fmt.Sprintf("%04d", count) != string(it.Value()) {
			t.Fatal(string(it.Value()))
		}
		val, err := sub1.Get(ro, []byte("hudel"+fmt.Sprintf("%04d", count)))
		if err != nil || string(val) != "Test1_"+fmt.Sprintf("%04d", count) {
			t.Fatal(err, count, string(val))
		}
		count++
	}
	if count != 100 {
		t.Fatalf("Iterator stopped too early or too late %v", count)
	}

	count = 99
	for it.SeekToLast(); it.Valid(); it.Prev() {
		if "hudel"+fmt.Sprintf("%04d", count) != string(it.Key()) {
			t.Fatal(string(it.Key()))
		}
		if "Test1_"+fmt.Sprintf("%04d", count) != string(it.Value()) {
			t.Fatal(string(it.Value()))
		}
		val, err := sub1.Get(ro, []byte("hudel"+fmt.Sprintf("%04d", count)))
		if err != nil || string(val) != "Test1_"+fmt.Sprintf("%04d", count) {
			t.Fatal(err, count, string(val))
		}
		count--
	}
	if count != -1 {
		t.Fatalf("Iterator stopped too early or too late %v", count)
	}
	it.Close()

	count = 0
	it = sub2.NewIterator(ro)
	for it.SeekToFirst(); it.Valid(); it.Next() {
		val, err := sub2.Get(ro, []byte("hudel"+fmt.Sprintf("%04d", count)))
		if err != nil || string(val) != "Test2_"+fmt.Sprintf("%04d", count) {
			t.Fatal(err)
		}
		count++
	}
	if count != 100 {
		t.Fatalf("Iterator stopped too early %v", count)
	}
	it.Close()

	ro.Close()
	wo.Close()
	db.Close()
}
Exemple #12
0
func TestHook(t *testing.T) {
	opts := levigo.NewOptions()
	levigo.DestroyDatabase("test.ldb", opts)
	// opts.SetCache(levigo.NewLRUCache(3<<30))
	opts.SetCreateIfMissing(true)
	db, err := levigo.Open("test.ldb", opts)
	if err != nil {
		t.Fatal(err)
	}

	ro := levigo.NewReadOptions()
	wo := levigo.NewWriteOptions()

	sub1 := Sublevel(db, "input")
	sub2 := Sublevel(db, "output")
	sub3 := Sublevel(db, "output2")
	sub1.Pre(func(key, value []byte, hook *Hook) {
		if value != nil {
			hook.Put(append([]byte("D_"), key...), value, sub2)
		} else {
			hook.Delete(append([]byte("D_"), key...), sub2)
		}
	})
	sub2.Pre(func(key, value []byte, hook *Hook) {
		if value != nil {
			hook.Put(append([]byte("X_"), key...), value, sub3)
		} else {
			hook.Delete(append([]byte("X_"), key...), sub3)
		}
	})

	post1Called := 0
	post3Called := 0

	sub1.Post(func(key, value []byte) {
		post1Called++
		val, err := sub1.Get(ro, key)
		if err != nil || bytes.Compare(val, value) != 0 {
			t.Fatal(err, string(val), string(value))
		}
	})

	sub3.Post(func(key, value []byte) {
		post3Called++
		val, err := sub3.Get(ro, key)
		if err != nil || bytes.Compare(val, value) != 0 {
			t.Fatal(err, "read=", string(val), "hook=", string(value))
		}
	})

	sub1.Put(wo, []byte("Name"), []byte("Horst"))
	val, err := sub1.Get(ro, []byte("Name"))
	if err != nil || string(val) != "Horst" {
		t.Fatal(err, string(val))
	}
	val, err = sub2.Get(ro, []byte("D_Name"))
	if err != nil || string(val) != "Horst" {
		t.Fatal(err, string(val))
	}
	val, err = sub3.Get(ro, []byte("X_D_Name"))
	if err != nil || string(val) != "Horst" {
		t.Fatal(err, string(val))
	}

	sub1.Delete(wo, []byte("Name"))
	val, err = sub1.Get(ro, []byte("Name"))
	if err != nil || val != nil {
		t.Fatal(err, string(val))
	}
	val, err = sub2.Get(ro, []byte("D_Name"))
	if err != nil || val != nil {
		t.Fatal(err, string(val))
	}
	val, err = sub3.Get(ro, []byte("X_D_Name"))
	if err != nil || val != nil {
		t.Fatal(err, string(val))
	}

	sub1.Put(wo, []byte("Color"), []byte("Red"))
	batch := sub1.NewWriteBatch()
	batch.Put([]byte("Sound"), []byte("Ping"))
	batch.Delete([]byte("Color"))
	sub1.Write(wo, batch)

	if post1Called != 5 {
		t.Fatal(post1Called)
	}
	if post3Called != 5 {
		t.Fatal(post3Called)
	}

	val, err = sub1.Get(ro, []byte("Color"))
	if err != nil || val != nil {
		t.Fatal(err, string(val))
	}
	val, err = sub2.Get(ro, []byte("D_Color"))
	if err != nil || val != nil {
		t.Fatal(err, string(val))
	}
	val, err = sub3.Get(ro, []byte("X_D_Color"))
	if err != nil || val != nil {
		t.Fatal(err, string(val))
	}

	val, err = sub1.Get(ro, []byte("Sound"))
	if err != nil || string(val) != "Ping" {
		t.Fatal(err, string(val))
	}
	val, err = sub2.Get(ro, []byte("D_Sound"))
	if err != nil || string(val) != "Ping" {
		t.Fatal(err, string(val))
	}
	val, err = sub3.Get(ro, []byte("X_D_Sound"))
	if err != nil || string(val) != "Ping" {
		t.Fatal(err, string(val))
	}

	ro.Close()
	wo.Close()
	db.Close()
}