コード例 #1
0
ファイル: pbf2json.go プロジェクト: piemapping/pbf2json
// write to leveldb immediately
func cacheStore(db *leveldb.DB, node *osmpbf.Node) {
	id, val := formatLevelDB(node)
	err := db.Put([]byte(id), []byte(val), nil)
	if err != nil {
		log.Fatal(err)
	}
}
コード例 #2
0
ファイル: main.go プロジェクト: peterjliu/gohn
func saveItem(db *leveldb.DB, it *gohn.Item) {
	pbmsg, err := proto.Marshal(it)
	check(err)
	key := []byte(strconv.Itoa(int(it.GetId())))
	err = db.Put(key, pbmsg, nil)
	check(err)
}
コード例 #3
0
ファイル: connectsdb.go プロジェクト: takumigashira/go-bus
/*
	/Put the DATABASE
	/Write the Key and Value
*/
func dbPut(dbObj *leveldb.DB, putData string, putDBKey []byte) int {
	var result int = 0

	putError := dbObj.Put([]byte(putDBKey), []byte(putData), nil)
	if putError != nil {
		result = 1
	}
	return result
}
コード例 #4
0
ファイル: quadstore.go プロジェクト: rlugojr/cayley
func setVersion(db *leveldb.DB, version int64, wo *opt.WriteOptions) error {
	buf := make([]byte, 8)
	order.PutUint64(buf, uint64(version))
	err := db.Put([]byte(versionKey), buf, wo)
	if err != nil {
		clog.Errorf("Couldn't write version!")
		return err
	}
	return nil
}
コード例 #5
0
func levelDbWrite(db *leveldb.DB,
	key uint64, offset uint32, size uint32) error {
	bytes := make([]byte, 16)
	util.Uint64toBytes(bytes[0:8], key)
	util.Uint32toBytes(bytes[8:12], offset)
	util.Uint32toBytes(bytes[12:16], size)
	if err := db.Put(bytes[0:8], bytes[8:16], nil); err != nil {
		return fmt.Errorf("failed to write leveldb: %v", err)
	}
	return nil
}
コード例 #6
0
func saveVideos(db *leveldb.DB, videos Videos) {
	for _, video := range videos {
		data, _ := db.Get([]byte(video.Url), nil)

		if len(data) == 0 {
			encoded, _ := json.Marshal(video)

			_ = db.Put([]byte(video.Url), []byte(encoded), nil)
		}
	}
}
コード例 #7
0
ファイル: wrapper.go プロジェクト: sidutta/cs733
func (rn *RaftNode) fSaveTerm(i int) {
	var currentTermDB *leveldb.DB
	var err error
	currentTermDB, err = leveldb.OpenFile(PATH+"/currentTerm", nil)
	for err != nil {
		currentTermDB, err = leveldb.OpenFile(PATH+"/currentTerm", nil)
	}
	// fmt.Println("sad", currentTermDB, reflect.TypeOf(currentTermDB), err, reflect.TypeOf(err))
	defer currentTermDB.Close()
	currentTermDB.Put([]byte(strconv.Itoa(rn.sm.ServerID)), []byte(strconv.Itoa(i)), nil)

}
コード例 #8
0
ファイル: wrapper.go プロジェクト: sidutta/cs733
func (rn *RaftNode) fSaveVotedFor(i int) {

	var votedForDB *leveldb.DB
	var err error
	votedForDB, err = leveldb.OpenFile(PATH+"/currentTerm", nil)
	for err != nil {
		votedForDB, err = leveldb.OpenFile(PATH+"/currentTerm", nil)
	}

	defer votedForDB.Close()
	votedForDB.Put([]byte(strconv.Itoa(rn.sm.ServerID)), []byte(strconv.Itoa(i)), nil)
}
コード例 #9
0
ファイル: log.go プロジェクト: myc880628/Raft
//Write entry to leveldb
func (e *LogItem) writeToDB(db *leveldb.DB) error {
	var network bytes.Buffer
	enc := gob.NewEncoder(&network)
	err := enc.Encode(e)
	if err != nil {
		panic("gob error: " + err.Error())
	}
	buf := make([]byte, 8)
	binary.LittleEndian.PutUint64(buf, e.Index)
	err = db.Put(buf, []byte(network.String()), nil)
	return err

}
コード例 #10
0
ファイル: js.go プロジェクト: teamSUDOs/localise-it
func InjectLevelDB(jsEngine *JSEngine, db *leveldb.DB) {
	jsEngine.Run("var db = {};")
	dbValue, _ := jsEngine.Get("db")
	dbObj := dbValue.Object()
	dbObj.Set("put", func(call otto.FunctionCall) otto.Value {
		key, err := call.Argument(0).ToString()
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		value, err := call.Argument(1).ToString()
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}

		err = db.Put([]byte(key), []byte(value), nil)
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		return otto.TrueValue()
	})
	dbObj.Set("get", func(call otto.FunctionCall) otto.Value {
		key, err := call.Argument(0).ToString()
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		data, err := db.Get([]byte(key), nil)
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		v, _ := otto.ToValue(string(data))
		return v
	})
	dbObj.Set("remove", func(call otto.FunctionCall) otto.Value {
		key, err := call.Argument(0).ToString()
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		err = db.Delete([]byte(key), nil)
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		return otto.TrueValue()
	})
}
コード例 #11
0
ファイル: server.go プロジェクト: saurabhjambhule/cs733
func doOperation(fileNm []byte, finalByt []byte, cmdTyp string, fileDB *leveldb.DB, wo *opt.WriteOptions) {

	var err error

	mutex.Lock()
	switch cmdTyp {
	case "write":
		err = fileDB.Put(fileNm, finalByt, nil)
	case "delete":
		err = fileDB.Delete([]byte(fileNm), wo)
	}
	mutex.Unlock()
	checkError(err)
}
コード例 #12
0
ファイル: server.go プロジェクト: pratyakshs/cs733
func doWrite(cmd *utils.Cmd, db *leveldb.DB) *utils.Cmd {

	dirMutex.Lock()
	defer dirMutex.Unlock()
	fi := dir[cmd.Filename]
	if fi != nil {
		if fi.Timer != nil {
			fi.Timer.Stop()
			fi.Timer = nil
		}
	} else {
		fi = &Metadata{}
	}

	if cmd.Exptime != 0 {
		dur := time.Duration(cmd.Exptime) * time.Second
		fi.Exptime = time.Now().Add(dur)
		// create timer for deletion
		timerFunc := func(name string, ver int64) func() {
			return func() {
				doDelete(&utils.Cmd{Type: "delete",
					Filename: name,
					Version:  ver}, db)
			}
		}(cmd.Filename, gversion)

		fi.Timer = time.AfterFunc(dur, timerFunc)

	}
	err := db.Put([]byte(cmd.Filename), cmd.Content, nil)

	if err != nil {
		log.Fatal(err)

	}

	gversion += 1
	fi.Version = gversion
	fi.Numbytes = cmd.Numbytes
	dir[cmd.Filename] = fi

	return &utils.Cmd{Type: "O", Version: gversion}
}
コード例 #13
0
ファイル: Raft.go プロジェクト: vibhor1403/KVStore
func applyLog(sc *ServerConfig, store *leveldb.DB) {
	timer := time.NewTimer(500 * time.Millisecond)
	var er error
	for {
		<-timer.C
		dbg4.Println(sc.basicCluster.Mypid, "logAplied", sc.commitIndex, sc.lastApplied, sc.votedFor)
		for i := sc.lastApplied + 1; i < sc.commitIndex; i++ {
			logMsg := sc.log[i].Msg.(string)
			splits := strings.Split(logMsg, " ")
			//dbg5.Println(splits)
			if splits[0] == "set" {
				er = store.Put([]byte(splits[1]), []byte(splits[2]), nil)
				//dbg5.Println("storing", er)
				if er == nil {
					sc.lastApplied += 1
				}
			}
		}
		timer = time.NewTimer(500 * time.Millisecond)
	}
}
コード例 #14
0
ファイル: main.go プロジェクト: mapleshadow/discosrv
func update(db *leveldb.DB, id protocol.DeviceID, addrs []address) {
	var newAddrs addressList

	val, err := db.Get(id[:], nil)
	if err == nil {
		newAddrs.UnmarshalXDR(val)
	}

nextAddr:
	for _, newAddr := range addrs {
		for i, exAddr := range newAddrs.addresses {
			if bytes.Compare(newAddr.ip, exAddr.ip) == 0 {
				newAddrs.addresses[i] = newAddr
				continue nextAddr
			}
		}
		newAddrs.addresses = append(newAddrs.addresses, newAddr)
	}

	db.Put(id[:], newAddrs.MarshalXDR(), nil)
}
コード例 #15
0
ファイル: main.go プロジェクト: mapleshadow/discosrv
func clean(statsLog io.Writer, db *leveldb.DB) {
	for {
		now := next(cacheLimitSeconds)
		nowSecs := now.Unix()

		var kept, deleted int64
		iter := db.NewIterator(nil, nil)
		for iter.Next() {
			var addrs addressList
			addrs.UnmarshalXDR(iter.Value())

			// Remove expired addresses
			newAddrs := addrs.addresses
			for i := 0; i < len(newAddrs); i++ {
				if nowSecs-newAddrs[i].seen > cacheLimitSeconds {
					newAddrs[i] = newAddrs[len(newAddrs)-1]
					newAddrs = newAddrs[:len(newAddrs)-1]
				}
			}

			// Delete empty records
			if len(newAddrs) == 0 {
				db.Delete(iter.Key(), nil)
				deleted++
				continue
			}

			// Update changed records
			if len(newAddrs) != len(addrs.addresses) {
				addrs.addresses = newAddrs
				db.Put(iter.Key(), addrs.MarshalXDR(), nil)
			}
			kept++
		}
		iter.Release()

		fmt.Fprintf(statsLog, "%d Kept:%d Deleted:%d Took:%0.04fs\n", nowSecs, kept, deleted, time.Since(now).Seconds())
	}
}
コード例 #16
0
// convertKeyFormat converts from the v0.12 to the v0.13 database format, to
// avoid having to do rescan. The change is in the key format for folder
// labels, so we basically just iterate over the database rewriting keys as
// necessary.
func convertKeyFormat(from, to *leveldb.DB) error {
	l.Infoln("Converting database key format")
	blocks, files, globals, unchanged := 0, 0, 0, 0

	dbi := newDBInstance(to)
	i := from.NewIterator(nil, nil)
	for i.Next() {
		key := i.Key()
		switch key[0] {
		case KeyTypeBlock:
			folder, file := oldFromBlockKey(key)
			folderIdx := dbi.folderIdx.ID([]byte(folder))
			hash := key[1+64:]
			newKey := blockKeyInto(nil, hash, folderIdx, file)
			if err := to.Put(newKey, i.Value(), nil); err != nil {
				return err
			}
			blocks++

		case KeyTypeDevice:
			newKey := dbi.deviceKey(oldDeviceKeyFolder(key), oldDeviceKeyDevice(key), oldDeviceKeyName(key))
			if err := to.Put(newKey, i.Value(), nil); err != nil {
				return err
			}
			files++

		case KeyTypeGlobal:
			newKey := dbi.globalKey(oldGlobalKeyFolder(key), oldGlobalKeyName(key))
			if err := to.Put(newKey, i.Value(), nil); err != nil {
				return err
			}
			globals++

		case KeyTypeVirtualMtime:
			// Cannot be converted, we drop it instead :(

		default:
			if err := to.Put(key, i.Value(), nil); err != nil {
				return err
			}
			unchanged++
		}
	}

	l.Infof("Converted %d blocks, %d files, %d globals (%d unchanged).", blocks, files, globals, unchanged)

	return nil
}
コード例 #17
0
ファイル: balbucket.go プロジェクト: xia0pin9/maldiff
func Worker(mapChan chan *Map, db *leveldb.DB, wg *sync.WaitGroup) {
	// Decreasing internal counter for wait-group as soon as goroutine finishes
	var fphash *[]byte
	var bucket float64
	var indexSplit []string
	var index string

	defer wg.Done()

	for input := range mapChan {
		indexSplit = strings.Split(input.index, "-")
		// FNV hash range [1, 2^32], divide into 2^4 buckets, each bucket has 2^28 unique values
		bucket = GetBucket(input.fphash)
		index = fmt.Sprintf("%.0f-%v-%v", bucket, indexSplit[2], indexSplit[3])
		//fmt.Println(index)
		//resChan <- &Result{index, md5, fphash, fpmap}
		//mapbyte, _ := json.Marshal(*fpmap)
		//    pkgbyte, _ := json.Marshal(*topPkgs)
		db.Put([]byte("h-"+index), *fphash, nil)
		//    db.Put([]byte("m-"+md5), mapbyte, nil)
		//    db.Put([]byte("p-"+md5+strconv.Itoa(numFunc)), pkgbyte, nil)
	}
}
コード例 #18
0
ファイル: fileserver.go プロジェクト: sidutta/cs733
// Command write
func write(conn net.Conn, input_bytes []byte, datadb *leveldb.DB, metadatadb *leveldb.DB, bytes_in_first_line int) {
	input_string := string(input_bytes)
	inputs := strings.Fields(input_string)
	filename := inputs[1]
	numbytes := inputs[2]
	var exptime string
	var exp string
	if len(inputs) == 4 {
		exptime = "2018-02-01 03:04:05 +0530 IST"
		exp = "0"
	} else {
		exp = inputs[3]
		delay, _ := strconv.Atoi(exp)
		exptime = time.Now().Add(time.Duration(delay) * time.Second).String()
	}
	mutex.Lock()
	prev_version_int, _, _, _, err := read_metadata(filename, metadatadb)
	new_version := ""
	if err == nil {
		if err != nil {
			log.Println("error in conversion: ", err)
		}
		saved_metadata := strconv.Itoa(prev_version_int+1) + " " + numbytes + " " + exptime + " " + exp
		err = metadatadb.Put([]byte(filename), []byte(saved_metadata), nil)
		if err != nil {
			log.Println("failed to add to database: ", err)
		}
		new_version = strconv.Itoa(prev_version_int + 1)
	} else {
		saved_metadata := strconv.Itoa(1) + " " + numbytes + " " + exptime + " " + exp
		err = metadatadb.Put([]byte(filename), []byte(saved_metadata), nil)
		if err != nil {
			log.Println("failed to add to database: ", err)
		}
		new_version = "1"
	}
	err = datadb.Put([]byte(filename), []byte(input_bytes[bytes_in_first_line:]), nil)
	mutex.Unlock()
	if err != nil {
		log.Println("failed to add to database: ", err)
	}
	response := "OK " + string(new_version) + "\r\n"
	// log.Println(response)
	some_int, err := conn.Write([]byte(response))
	if err != nil {
		log.Println("failed to reply back: ", err, some_int)
	}
}
コード例 #19
0
ファイル: generate_map.go プロジェクト: xia0pin9/maldiff
func Worker(indir string, md5Chan chan string, db *leveldb.DB, wg *sync.WaitGroup) {
	// Decreasing internal counter for wait-group as soon as goroutine finishes
	var out io.ReadCloser
	var topPkgs *map[string]struct{}
	var fphash *[]byte
	var fpmap *Fpmap
	var numFunc int
	var inpath, command, index string
	var err error
	var nbit uint
	var nlogbit float64
	done := make(chan error, 1)

	defer wg.Done()

	for md5 := range md5Chan {
		inpath = fmt.Sprintf("%s", path.Join(indir, md5+".dex"))
		command = fmt.Sprintf("%s -d -l plain %s", path.Join(libpath, "dexdump"), inpath)
		cmd := exec.Command("sh", "-c", command)
		out, err = cmd.StdoutPipe()
		if err != nil {
			Error.Println("Error reading cmd", md5, err)
		}

		if err = cmd.Start(); err != nil {
			Error.Println("Error starting Cmd", err, md5)
		}

		numFunc, topPkgs, fphash, fpmap = GenerateHash(GetCode(out))

		go func() {
			done <- cmd.Wait()
		}()
		select {
		case <-time.After(time.Second * 180):
			if err = cmd.Process.Kill(); err != nil {
				Error.Println("Failed to kill", err)
			}
			<-done
			Error.Println("Error exec time out ", md5)
		case err = <-done:
			if err != nil {
				Error.Println("Error waiting for Cmd", err, md5)
			}
		}

		nbit = bitcount(*fphash)
		if nbit > 0 {
			nlogbit = math.Log2(float64(nbit)) + 1
			index = fmt.Sprintf("%.0f-%v-%v", nlogbit, nbit, md5)
			//fmt.Println(index)
			//resChan <- &Result{index, md5, fphash, fpmap}
			mapbyte, _ := json.Marshal(*fpmap)
			pkgbyte, _ := json.Marshal(*topPkgs)
			db.Put([]byte("h-"+index), *fphash, nil)
			db.Put([]byte("m-"+md5), mapbyte, nil)
			db.Put([]byte("p-"+md5+strconv.Itoa(numFunc)), pkgbyte, nil)
		} else {
			db.Put([]byte("h-Inf-0-"+md5), nil, nil)
			Info.Println("Empty fphash:", md5)
		}
	}
}
コード例 #20
0
ファイル: server.go プロジェクト: aakashdeshpande/cs733
func handleConnection(conn net.Conn, db *leveldb.DB, files *leveldb.DB, expiry *leveldb.DB, mutex *sync.RWMutex) {

	defer conn.Close() // make sure to close the connection even if we panic.
	// Make a buffer to hold incoming data with size bufferSize
	var bufferSize int64 = 1024
	buf := make([]byte, bufferSize)
	var fileName string
	var sLarge string = "" // Initial command string and number of bytes read with the command
	var bufferRead int = 0

	// Iterate over different commands, takes residual string as input
OUTER:
	for {

		// Check residual string from previous command
		buf = []byte(sLarge)
		bufferRead = len([]byte(sLarge))

		// Read the incoming connection into the buffer
		// Warning : We might not get the complete message
		for {
			if strings.Contains(sLarge, "\r\n") || len(sLarge) > 100000 {
				break
			}
			buf = make([]byte, bufferSize)
			n, _ := conn.Read(buf)
			sLarge += string(buf[:n]) // Read till we get the complete command
			bufferRead = n
		}

		//fmt.Println(sLarge, len(sLarge))
		s := strings.Split(sLarge, "\r\n") // Isolate main command from input
		commands := strings.Split(s[0], " ")
		// Remove extra input apart from basic command string
		var remainder int64 = int64(len([]byte(sLarge)) - len([]byte(s[0]+"\r\n")))
		sLarge = sLarge[len(s[0]+"\r\n"):]

		if len(commands) < 2 {
			conn.Write([]byte("ERR_CMD_ERR\r\n"))
			continue OUTER
		} else {
			fileName = commands[1]
		}

		if commands[0] == "read" {
			mutex.RLock()
			file, err := files.Get([]byte(fileName), nil)
			if err != nil {
				conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
				mutex.RUnlock()
				continue OUTER
			}
			//Check expiry
			var diff int64 = 0
			expbytes, err := expiry.Get([]byte(fileName), nil)
			if err == nil {
				exp, _ := strconv.ParseInt(string(expbytes), 10, 64)
				if exp != -1 {
					diff = exp - time.Now().Unix()
				}
			}
			if diff < 0 {
				conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
				mutex.RUnlock()
				continue OUTER
			}

			// Check version
			var version int64 = 0
			data, err := db.Get([]byte(fileName), nil)
			if err == nil {
				version, err = strconv.ParseInt(string(data), 10, 64)
			}
			mutex.RUnlock()

			conn.Write([]byte("CONTENTS " + strconv.FormatInt(version, 10) + " " + strconv.Itoa(len(file)-2) + " " + strconv.FormatInt(diff, 10) + "\r\n"))

			for i := 0; i < len(file)-2; i++ {
				_, err = conn.Write(file[i : i+1])
			}
			_, err = conn.Write([]byte("\r\n"))

			if err != nil && err != io.EOF {
				conn.Write([]byte("ERR_INTERNAL\r\n"))
			}
		} else if commands[0] == "delete" {
			err := files.Delete([]byte(fileName), nil) // Remove entry from database
			//err := os.Remove(fileName)
			if err != nil {
				conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
			} else {
				db.Delete([]byte(fileName), nil)     // Remove entry from database
				expiry.Delete([]byte(fileName), nil) // Remove entry from database
				conn.Write([]byte("OK\r\n"))
			}
		} else if commands[0] == "write" {
			sLarge = ""

			if len(commands) < 3 {
				conn.Write([]byte("ERR_CMD_ERR\r\n"))
				continue OUTER
			}

			// NUmber of bytes to be written to the file
			fileSize, err := strconv.ParseInt(commands[2], 10, 64)
			if err != nil {
				conn.Write([]byte("ERR_CMD_ERR\r\n"))
				continue OUTER
			}
			fileSize = fileSize + int64(len([]byte("\r\n")))

			file := write(conn, fileSize, remainder, bufferRead, buf, &sLarge)

			mutex.Lock()

			err = files.Put([]byte(fileName), file, nil)
			// Get file version
			var version int64 = 0
			data, err := db.Get([]byte(fileName), nil)
			if err == nil {
				version, err = strconv.ParseInt(string(data), 10, 64)
			}

			err = db.Put([]byte(fileName), []byte(strconv.FormatInt(version+1, 10)), nil)
			conn.Write([]byte("OK " + strconv.FormatInt(version+1, 10) + "\r\n"))

			// Write expiry time
			var exp int64 = 0
			if len(commands) > 3 {
				exp, _ = strconv.ParseInt(commands[3], 10, 64)
			}
			if exp == 0 {
				err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(-1, 10)), nil)
			} else {
				err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(time.Now().Unix()+exp, 10)), nil)
			}

			mutex.Unlock()

		} else if commands[0] == "cas" {
			sLarge = ""

			if len(commands) < 4 {
				conn.Write([]byte("ERR_CMD_ERR\r\n"))
				continue OUTER
			}

			// Number of bytes to be written to file
			fileSize, err := strconv.ParseInt(commands[3], 10, 64)
			if err != nil {
				conn.Write([]byte("ERR_CMD_ERR\r\n"))
				continue OUTER
			}
			fileSize = fileSize + int64(len([]byte("\r\n")))

			file := write(conn, fileSize, remainder, bufferRead, buf, &sLarge)

			mutex.Lock()

			// Get file version
			var version int64 = 0
			data, err := db.Get([]byte(fileName), nil)
			if err == nil {
				version, err = strconv.ParseInt(string(data), 10, 64)
			}

			if err != nil {
				conn.Write([]byte("ERR_INTERNAL\r\n"))
				mutex.Unlock()
				return
			}

			fileVersion, err := strconv.ParseInt(commands[2], 10, 64)

			// Check for version match
			if version != fileVersion {
				conn.Write([]byte("ERR_VERSION " + strconv.FormatInt(version, 10) + "\r\n"))
				mutex.Unlock()
				continue OUTER
			}

			err = files.Put([]byte(fileName), file, nil)

			err = db.Put([]byte(fileName), []byte(strconv.FormatInt(version, 10)), nil)
			conn.Write([]byte("OK " + strconv.FormatInt(version, 10) + "\r\n"))

			// Write expiry time
			var exp int64 = 0
			if len(commands) > 4 {
				exp, _ = strconv.ParseInt(commands[4], 10, 64)
			}
			if exp == 0 {
				err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(-1, 10)), nil)
			} else {
				err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(time.Now().Unix()+exp, 10)), nil)
			}

			mutex.Unlock()

		} else {
			conn.Write([]byte("ERR_CMD_ERR\r\n"))
			continue OUTER
		}

	}
}
コード例 #21
0
ファイル: migrate.go プロジェクト: rlugojr/cayley
func upgrade1To2(db *leveldb.DB) error {
	fmt.Println("Upgrading v1 to v2...")

	type v1IndexEntry struct {
		Subject   string  `json:"subject"`
		Predicate string  `json:"predicate"`
		Object    string  `json:"object"`
		Label     string  `json:"label,omitempty"`
		History   []int64 `json:"History"`
	}

	type v1ValueData struct {
		Name string `json:"Name"`
		Size int64  `json:"Size"`
	}

	var (
		spoPref = []byte{spo[0].Prefix(), spo[1].Prefix()}
		ospPref = []byte{osp[0].Prefix(), osp[1].Prefix()}
		posPref = []byte{pos[0].Prefix(), pos[1].Prefix()}
		cpsPref = []byte{cps[0].Prefix(), cps[1].Prefix()}
	)

	{
		fmt.Println("Upgrading bucket z")
		it := db.NewIterator(&util.Range{Start: []byte{'z'}, Limit: []byte{'z' + 1}}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			var val v1ValueData
			if err := json.Unmarshal(v, &val); err != nil {
				return err
			}
			node := proto.NodeData{
				Size:  val.Size,
				Value: proto.MakeValue(quad.Raw(val.Name)),
			}
			nv, err := node.Marshal()
			if err != nil {
				return err
			}
			if err = db.Put(k, nv, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	for _, pref := range [4][]byte{spoPref, ospPref, posPref, cpsPref} {
		fmt.Println("Upgrading bucket", string(pref))
		end := []byte{pref[0], pref[1] + 1}
		it := db.NewIterator(&util.Range{Start: pref, Limit: end}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			var entry v1IndexEntry
			if err := json.Unmarshal(v, &entry); err != nil {
				return err
			}
			var h proto.HistoryEntry
			h.History = make([]uint64, len(entry.History))
			for i, id := range entry.History {
				h.History[i] = uint64(id)
			}
			nv, err := h.Marshal()
			if err != nil {
				return err
			}
			if err = db.Put(k, nv, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	{
		fmt.Println("Upgrading bucket d")
		it := db.NewIterator(&util.Range{Start: []byte{'d'}, Limit: []byte{'d' + 1}}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			id, err := strconv.ParseInt(string(k[1:]), 16, 64)
			if err != nil {
				return err
			}
			nk := createDeltaKeyFor(id)
			var val graph.Delta
			if err := json.Unmarshal(v, &val); err != nil {
				return err
			}
			p := deltaToProto(val)
			nv, err := p.Marshal()
			if err != nil {
				return err
			}
			b := &leveldb.Batch{}
			b.Put(nk, nv)
			b.Delete(k)
			if err = db.Write(b, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	return nil
}
コード例 #22
0
ファイル: hivetool.go プロジェクト: reactor10/hivemind
func writeToDb(db *leveldb.DB, k string, v string) {
    log.Println(k + " - " + v)
    db.Put([]byte(k), []byte(v), nil)
}
コード例 #23
0
ファイル: main.go プロジェクト: tgulacsi/leveldb-tools
func main() {
	rootCmd := &cobra.Command{
		Short: "leveldb-tools",
	}
	var db *leveldb.DB
	openDB := func(cmd *cobra.Command, args []string) {
		if len(args) != 1 {
			rootCmd.Usage()
			os.Exit(1)
		}
		var err error
		opts := &opt.Options{ErrorIfMissing: cmd.Use == "dump", Strict: opt.StrictAll}
		if db, err = leveldb.OpenFile(args[0], opts); err != nil {
			log.Fatal(err)
		}
	}

	dumpCmd := &cobra.Command{
		Use:   "dump",
		Short: "dump database",
		Run: func(cmd *cobra.Command, args []string) {
			defer db.Close()
			defer os.Stdout.Close()
			w := bufio.NewWriter(os.Stdout)
			defer w.Flush()
			iter := db.NewIterator(nil, &opt.ReadOptions{Strict: opt.StrictAll, DontFillCache: true})
			defer iter.Release()
			sep := sep[:]
			for iter.Next() {
				k, v := iter.Key(), iter.Value()
				w.WriteString(fmt.Sprintf("+%d,%d:", len(k), len(v)))
				w.Write(k)
				w.Write(sep)
				w.Write(v)
				if err := w.WriteByte('\n'); err != nil {
					log.Fatal(err)
				}
			}
			if err := iter.Error(); err != nil {
				log.Fatal(err)
			}
		},
		PersistentPreRun: openDB,
	}
	rootCmd.AddCommand(dumpCmd)

	loadCmd := &cobra.Command{
		Use:   "load",
		Short: "load database",
		Run: func(cmd *cobra.Command, args []string) {
			defer func() {
				if err := db.Close(); err != nil {
					log.Fatal(err)
				}
			}()
			r := bufio.NewReader(os.Stdin)
			var lk, lv int
			var k, v []byte
			sepLen := len(sep)
			sep := sep[:]
			for {
				if _, err := fmt.Fscanf(r, "+%d,%d:", &lk, &lv); err != nil {
					if err == io.EOF || err == io.ErrUnexpectedEOF {
						break
					}
					log.Fatal(err)
				}
				if cap(k) < lk {
					k = make([]byte, lk*2)
				}
				if cap(v) < lv+sepLen+1 {
					v = make([]byte, lv*2+2)
				}
				n, err := io.ReadFull(r, k[:lk])
				if err != nil {
					log.Fatal(err)
				}
				k = k[:n]
				if n, err = io.ReadFull(r, v[:lv+sepLen+1]); err != nil {
					log.Fatal(err)
				}
				if !bytes.Equal(sep, v[:sepLen]) {
					log.Fatal("awaited %q, got %q", sep, v)
				}
				v = v[:n]
				if v[n-1] != '\n' {
					log.Fatal("should end with EOL, got %q", v)
				}
				v = v[2 : n-1]
				if err := db.Put(k, v, nil); err != nil {
					log.Fatal(err)
				}
			}
		},
		PersistentPreRun: openDB,
	}
	rootCmd.AddCommand(loadCmd)
	rootCmd.Execute()
}