Beispiel #1
0
func saveItem(db *leveldb.DB, it *gohn.Item) {
	pbmsg, err := proto.Marshal(it)
	check(err)
	key := []byte(strconv.Itoa(int(it.GetId())))
	err = db.Put(key, pbmsg, nil)
	check(err)
}
Beispiel #2
0
func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
	k := globalKey(folder, file)
	bs, err := db.Get(k, nil)
	if err == leveldb.ErrNotFound {
		return nil
	}
	if err != nil {
		panic(err)
	}

	var vl versionList
	err = vl.UnmarshalXDR(bs)
	if err != nil {
		panic(err)
	}

	var devices []protocol.DeviceID
	for _, v := range vl.versions {
		if !v.version.Equal(vl.versions[0].version) {
			break
		}
		n := protocol.DeviceIDFromBytes(v.device)
		devices = append(devices, n)
	}

	return devices
}
Beispiel #3
0
func ldbListFolders(db *leveldb.DB) []string {
	runtime.GC()

	start := []byte{keyTypeGlobal}
	limit := []byte{keyTypeGlobal + 1}
	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	defer snap.Release()
	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	folderExists := make(map[string]bool)
	for dbi.Next() {
		folder := string(globalKeyFolder(dbi.Key()))
		if !folderExists[folder] {
			folderExists[folder] = true
		}
	}

	folders := make([]string, 0, len(folderExists))
	for k := range folderExists {
		folders = append(folders, k)
	}

	sort.Strings(folders)
	return folders
}
Beispiel #4
0
// flush a leveldb batch to database and reset batch to 0
func cacheFlush(db *leveldb.DB, batch *leveldb.Batch) {
	err := db.Write(batch, nil)
	if err != nil {
		log.Fatal(err)
	}
	batch.Reset()
}
Beispiel #5
0
func cacheLookup(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) {

	var container []map[string]string

	for _, each := range way.NodeIDs {
		stringid := strconv.FormatInt(each, 10)

		data, err := db.Get([]byte(stringid), nil)
		if err != nil {
			log.Println("denormalize failed for way:", way.ID, "node not found:", stringid)
			return container, err
		}

		s := string(data)
		spl := strings.Split(s, ":")

		latlon := make(map[string]string)
		lat, lon := spl[0], spl[1]
		latlon["lat"] = lat
		latlon["lon"] = lon

		container = append(container, latlon)

	}

	return container, nil

	// fmt.Println(way.NodeIDs)
	// fmt.Println(container)
	// os.Exit(1)
}
Beispiel #6
0
// write to leveldb immediately
func cacheStore(db *leveldb.DB, node *osmpbf.Node) {
	id, val := formatLevelDB(node)
	err := db.Put([]byte(id), []byte(val), nil)
	if err != nil {
		log.Fatal(err)
	}
}
Beispiel #7
0
func read_metadata(filename string, metadatadb *leveldb.DB) (version int, numbytes int, exptime time.Time, exp int, err error) {
	data, err2 := metadatadb.Get([]byte(filename), nil)
	// log.Println(data, filename, err2)
	err = err2
	if err != nil {
		version = 0
		numbytes = 0
		exptime = time.Now()
		exp = 0
	} else {
		line := string(data)
		fields := strings.Fields(line)
		version, _ = strconv.Atoi(fields[0])
		numbytes, _ = strconv.Atoi(fields[1])
		if len(fields) == 7 {
			exptime_str := fields[2] + " " + fields[3] + " " + fields[4] + " " + fields[5]
			layout := "2006-01-02 15:04:05 -0700 MST"
			exptime, _ = time.Parse(layout, exptime_str)
			exp, _ = strconv.Atoi(fields[6])
		} else {
			log.Println("why here?", string(data), string(filename))
		}
	}
	return
}
Beispiel #8
0
//从levelDB中载入列表判断是否近期执行
func WalkTasks(base, span int64, db *leveldb.DB) {
	var task Task
	var err error
	var ok bool
	timer.DropMap = nil
	timer.DropMap = make(map[string]bool)
	s := time.Now()
	iter := db.NewIterator(nil, nil)
	parseMap := make(map[string][]string, 1024)
	for iter.Next() {
		key := string(iter.Key())
		val := string(iter.Value())
		if _, ok = parseMap[val]; ok == false {
			parseMap[val] = make([]string, 0)
		}
		parseMap[val] = append(parseMap[val], key)
	}
	iter.Release()
	inWalking = true
	for val, keys := range parseMap {
		task, err = parseTask(val)
		if err == nil {
			task.Base = base
			ret := task.Dispatch(span, true)
			for _, key := range keys {
				task.Key = key
				timer.Add(ret, key)
			}
		}
	}
	inWalking = false
	parseMap = nil
	e := time.Now()
	logPrintln(e.Sub(s))
}
Beispiel #9
0
func clearItems(db *leveldb.DB) error {
	snap, err := db.GetSnapshot()
	if err != nil {
		return err
	}
	defer snap.Release()

	// Iterate over k2

	it := snap.NewIterator(util.BytesPrefix([]byte{1}), nil)
	defer it.Release()

	batch := new(leveldb.Batch)
	for it.Next() {
		k1 := it.Key()
		k2 := it.Value()

		// k2 should exist
		_, err := snap.Get(k2, nil)
		if err != nil {
			return err
		}

		// Delete the k1 => k2 mapping first
		batch.Delete(k1)
		// Then the k2 => data mapping
		batch.Delete(k2)
	}
	if testing.Verbose() {
		log.Printf("batch write (clear) %p", batch)
	}
	return db.Write(batch, nil)
}
Beispiel #10
0
func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []byte, f protocol.FileInfoTruncated) bool) {
	runtime.GC()

	start := deviceKey(folder, nil, nil)                                                  // before all folder/device files
	limit := deviceKey(folder, protocol.LocalDeviceID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	if debugDB {
		l.Debugf("created snapshot %p", snap)
	}
	defer func() {
		if debugDB {
			l.Debugf("close snapshot %p", snap)
		}
		snap.Release()
	}()

	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	for dbi.Next() {
		device := deviceKeyDevice(dbi.Key())
		var f protocol.FileInfoTruncated
		err := f.UnmarshalXDR(dbi.Value())
		if err != nil {
			panic(err)
		}
		if cont := fn(device, f); !cont {
			return
		}
	}
}
Beispiel #11
0
func ldbWithAllRepoTruncated(db *leveldb.DB, repo []byte, fn func(node []byte, f protocol.FileInfoTruncated) bool) {
	defer runtime.GC()

	start := nodeKey(repo, nil, nil)                                                // before all repo/node files
	limit := nodeKey(repo, protocol.LocalNodeID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	defer snap.Release()
	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	for dbi.Next() {
		node := nodeKeyNode(dbi.Key())
		var f protocol.FileInfoTruncated
		err := f.UnmarshalXDR(dbi.Value())
		if err != nil {
			panic(err)
		}
		if cont := fn(node, f); !cont {
			return
		}
	}
}
Beispiel #12
0
func ldbListRepos(db *leveldb.DB) []string {
	defer runtime.GC()

	start := []byte{keyTypeGlobal}
	limit := []byte{keyTypeGlobal + 1}
	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	defer snap.Release()
	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	repoExists := make(map[string]bool)
	for dbi.Next() {
		repo := string(globalKeyRepo(dbi.Key()))
		if !repoExists[repo] {
			repoExists[repo] = true
		}
	}

	repos := make([]string, 0, len(repoExists))
	for k := range repoExists {
		repos = append(repos, k)
	}

	sort.Strings(repos)
	return repos
}
Beispiel #13
0
//Function: getFile
//Checkes whether file exists or not
//Parameters:
//    fileName - file name
//    fileDB - database connection
//Return:
//	  true - file found
//	  false - file not found
//	  file itself
func getFile(fileName string, fileDB *leveldb.DB) (bool, []byte, iterator.Iterator) {

	var flag int = 0
	var val []byte

	iter := fileDB.NewIterator(nil, nil)

	//doOperation([]byte(fileNm), nil, "delete", fileDB, wo)

	for iter.Next() {
		key := iter.Key()
		if string(key) == fileName {
			val = iter.Value()
			flag = 1
			break
		}
	}

	if flag == 1 {
		return true, val, iter
	} else {
		return false, nil, iter
	}

}
func clearItems(db *leveldb.DB) error {
	snap, err := db.GetSnapshot()
	if err != nil {
		return err
	}
	defer snap.Release()

	// Iterate from the start of k2 space to the end
	it := snap.NewIterator(&util.Range{Start: []byte{2}, Limit: []byte{2, 0xff, 0xff, 0xff, 0xff}}, nil)
	defer it.Release()

	batch := &leveldb.Batch{}
	for it.Next() {
		k2 := it.Key()
		k1 := it.Value()

		// k1 should exist
		_, err := snap.Get(k1, nil)
		if err != nil {
			return err
		}

		// Delete the k2 => k1 mapping first
		batch.Delete(k2)
		// Then the k1 => key mapping
		batch.Delete(k1)
	}
	return db.Write(batch, nil)
}
Beispiel #15
0
/*
	/Read the DATABASE
	/Read the Key and Value by Key
*/
func dbRead(dbObj *leveldb.DB, readDBKey []byte) []byte {
	data, readError := dbObj.Get([]byte(readDBKey), nil)
	if readError != nil {
		fmt.Println("DB Read ERROR")
	}
	return data
}
Beispiel #16
0
func ldbAvailability(db *leveldb.DB, repo, file []byte) []protocol.NodeID {
	k := globalKey(repo, file)
	bs, err := db.Get(k, nil)
	if err == leveldb.ErrNotFound {
		return nil
	}
	if err != nil {
		panic(err)
	}

	var vl versionList
	err = vl.UnmarshalXDR(bs)
	if err != nil {
		panic(err)
	}

	var nodes []protocol.NodeID
	for _, v := range vl.versions {
		if v.version != vl.versions[0].version {
			break
		}
		var n protocol.NodeID
		copy(n[:], v.node)
		nodes = append(nodes, n)
	}

	return nodes
}
Beispiel #17
0
func ldbListFolders(db *leveldb.DB) []string {
	runtime.GC()

	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	l.Debugf("created snapshot %p", snap)
	defer func() {
		l.Debugf("close snapshot %p", snap)
		snap.Release()
	}()

	dbi := snap.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
	defer dbi.Release()

	folderExists := make(map[string]bool)
	for dbi.Next() {
		folder := string(globalKeyFolder(dbi.Key()))
		if !folderExists[folder] {
			folderExists[folder] = true
		}
	}

	folders := make([]string, 0, len(folderExists))
	for k := range folderExists {
		folders = append(folders, k)
	}

	sort.Strings(folders)
	return folders
}
Beispiel #18
0
// Command read
func read(conn net.Conn, input_bytes []byte, datadb *leveldb.DB, metadatadb *leveldb.DB) {
	input_string := string(input_bytes)
	inputs := strings.Fields(input_string)
	filename := inputs[1]

	mutex.RLock()
	version, numbytes, exptime, exp, err1 := read_metadata(filename, metadatadb)

	if err1 == nil {
		if time.Now().After(exptime) {
			mutex.RUnlock()
			conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n")) // content has expired
		} else {
			data, err2 := datadb.Get([]byte(filename), nil)
			mutex.RUnlock()
			if err2 != nil {
				log.Println("error in conversion: ", err1)
			}
			version_str := strconv.Itoa(version)
			numbytes_str := strconv.Itoa(numbytes)

			exp_str := "0"
			if exp != 0 {
				exp_str = strconv.Itoa(int(exptime.Sub(time.Now()).Seconds()))
			}
			response := append([]byte("CONTENTS "+version_str+" "+numbytes_str+" "+exp_str+" \r\n"), data...)
			// log.Println(response)
			conn.Write(response)
			// log.Println(response)
		}
	} else {
		mutex.RUnlock()
		conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
	}
}
Beispiel #19
0
func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
	start := deviceKey(folder, device, nil)                            // before all folder/device files
	limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	if debugDB {
		l.Debugf("created snapshot %p", snap)
	}
	defer func() {
		if debugDB {
			l.Debugf("close snapshot %p", snap)
		}
		snap.Release()
	}()

	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	for dbi.Next() {
		f, err := unmarshalTrunc(dbi.Value(), truncate)
		if err != nil {
			panic(err)
		}
		if cont := fn(f); !cont {
			return
		}
	}
}
Beispiel #20
0
func BitIndexes(bdb *leveldb.DB, bmd5s []string) map[uint]struct{} {
	wg := new(sync.WaitGroup)
	res := make(map[uint]struct{})
	bitChan := make(chan uint, 1000)

	go func() {
		for bit := range bitChan {
			if _, ok := res[bit]; !ok {
				res[bit] = struct{}{}
			}
		}
	}()

	for _, bmd5 := range bmd5s {
		wg.Add(1)
		go func(bmd5 string) {
			var fpmap Fpmap

			data, _ := bdb.Get([]byte("m-"+bmd5), nil)
			if err := json.Unmarshal(data, &fpmap); err != nil {
				Error.Println("Error decoding fpmap object", bmd5)
			}
			for _, index := range fpmap {
				bitChan <- index.bindex
			}
		}(bmd5)
	}
	wg.Wait()
	close(bitChan)
	return res
}
Beispiel #21
0
func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn fileIterator) {
	runtime.GC()

	start := globalKey(folder, nil)
	limit := globalKey(folder, []byte{0xff, 0xff, 0xff, 0xff})
	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	if debugDB {
		l.Debugf("created snapshot %p", snap)
	}
	defer func() {
		if debugDB {
			l.Debugf("close snapshot %p", snap)
		}
		snap.Release()
	}()

	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	for dbi.Next() {
		var vl versionList
		err := vl.UnmarshalXDR(dbi.Value())
		if err != nil {
			panic(err)
		}
		if len(vl.versions) == 0 {
			l.Debugln(dbi.Key())
			panic("no versions?")
		}
		name := globalKeyName(dbi.Key())
		fk := deviceKey(folder, vl.versions[0].device, name)
		if debugDB {
			l.Debugf("snap.Get %p %x", snap, fk)
		}
		bs, err := snap.Get(fk, nil)
		if err != nil {
			l.Debugf("folder: %q (%x)", folder, folder)
			l.Debugf("key: %q (%x)", dbi.Key(), dbi.Key())
			l.Debugf("vl: %v", vl)
			l.Debugf("vl.versions[0].device: %x", vl.versions[0].device)
			l.Debugf("name: %q (%x)", name, name)
			l.Debugf("fk: %q", fk)
			l.Debugf("fk: %x %x %x", fk[1:1+64], fk[1+64:1+64+32], fk[1+64+32:])
			panic(err)
		}

		f, err := unmarshalTrunc(bs, truncate)
		if err != nil {
			panic(err)
		}

		if cont := fn(f); !cont {
			return
		}
	}
}
Beispiel #22
0
func dbEmpty(db *leveldb.DB) bool {
	iter := db.NewIterator(nil, nil)
	defer iter.Release()
	if iter.Next() {
		return false
	}
	return true
}
Beispiel #23
0
func get(db *leveldb.DB, id protocol.DeviceID) []address {
	var addrs addressList
	val, err := db.Get(id[:], nil)
	if err == nil {
		addrs.UnmarshalXDR(val)
	}
	return addrs.addresses
}
Beispiel #24
0
func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
	defer runtime.GC()

	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	l.Debugf("created snapshot %p", snap)
	defer func() {
		l.Debugf("close snapshot %p", snap)
		snap.Release()
	}()

	start := globalKey(folder, nil)
	limit := globalKey(folder, []byte{0xff, 0xff, 0xff, 0xff})
	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	batch := new(leveldb.Batch)
	l.Debugf("new batch %p", batch)

	var fk []byte
	for dbi.Next() {
		gk := dbi.Key()
		var vl versionList
		err := vl.UnmarshalXDR(dbi.Value())
		if err != nil {
			panic(err)
		}

		// Check the global version list for consistency. An issue in previous
		// versions of goleveldb could result in reordered writes so that
		// there are global entries pointing to no longer existing files. Here
		// we find those and clear them out.

		name := globalKeyName(gk)
		var newVL versionList
		for _, version := range vl.versions {
			fk = deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
			l.Debugf("snap.Get %p %x", snap, fk)
			_, err := snap.Get(fk, nil)
			if err == leveldb.ErrNotFound {
				continue
			}
			if err != nil {
				panic(err)
			}
			newVL.versions = append(newVL.versions, version)
		}

		if len(newVL.versions) != len(vl.versions) {
			l.Infof("db repair: rewriting global version list for %x %x", gk[1:1+64], gk[1+64:])
			batch.Put(dbi.Key(), newVL.MustMarshalXDR())
		}
	}
	l.Debugf("db check completed for %q", folder)
	db.Write(batch, nil)
}
Beispiel #25
0
func getVersion(db *leveldb.DB) (int64, error) {
	data, err := db.Get([]byte(versionKey), nil)
	if err == leveldb.ErrNotFound {
		return nilDataVersion, nil
	} else if len(data) != 8 {
		return 0, fmt.Errorf("version value format is unknown")
	}
	return int64(order.Uint64(data)), nil
}
Beispiel #26
0
/*
	/Put the DATABASE
	/Write the Key and Value
*/
func dbPut(dbObj *leveldb.DB, putData string, putDBKey []byte) int {
	var result int = 0

	putError := dbObj.Put([]byte(putDBKey), []byte(putData), nil)
	if putError != nil {
		result = 1
	}
	return result
}
Beispiel #27
0
func setVersion(db *leveldb.DB, version int64, wo *opt.WriteOptions) error {
	buf := make([]byte, 8)
	order.PutUint64(buf, uint64(version))
	err := db.Put([]byte(versionKey), buf, wo)
	if err != nil {
		clog.Errorf("Couldn't write version!")
		return err
	}
	return nil
}
Beispiel #28
0
/*
/Read the DATABASE
/Read the All Key and Value by Key
*/
func dbAllRead(dbObj *leveldb.DB, readDBKey []byte) {

	//create Iterator
	iter := dbObj.NewIterator(nil, nil)

	//Read to the end of the Key
	for iter.Seek(readDBKey); iter.Valid(); iter.Next() {
		fmt.Println(string(iter.Value()))
	}
}
func saveVideos(db *leveldb.DB, videos Videos) {
	for _, video := range videos {
		data, _ := db.Get([]byte(video.Url), nil)

		if len(data) == 0 {
			encoded, _ := json.Marshal(video)

			_ = db.Put([]byte(video.Url), []byte(encoded), nil)
		}
	}
}
func levelDbWrite(db *leveldb.DB,
	key uint64, offset uint32, size uint32) error {
	bytes := make([]byte, 16)
	util.Uint64toBytes(bytes[0:8], key)
	util.Uint32toBytes(bytes[8:12], offset)
	util.Uint32toBytes(bytes[12:16], size)
	if err := db.Put(bytes[0:8], bytes[8:16], nil); err != nil {
		return fmt.Errorf("failed to write leveldb: %v", err)
	}
	return nil
}