Beispiel #1
0
func ldbDropRepo(db *leveldb.DB, repo []byte) {
	defer runtime.GC()

	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	defer snap.Release()

	// Remove all items related to the given repo from the node->file bucket
	start := []byte{keyTypeNode}
	limit := []byte{keyTypeNode + 1}
	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	for dbi.Next() {
		itemRepo := nodeKeyRepo(dbi.Key())
		if bytes.Compare(repo, itemRepo) == 0 {
			db.Delete(dbi.Key(), nil)
		}
	}
	dbi.Release()

	// Remove all items related to the given repo from the global bucket
	start = []byte{keyTypeGlobal}
	limit = []byte{keyTypeGlobal + 1}
	dbi = snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	for dbi.Next() {
		itemRepo := globalKeyRepo(dbi.Key())
		if bytes.Compare(repo, itemRepo) == 0 {
			db.Delete(dbi.Key(), nil)
		}
	}
	dbi.Release()
}
Beispiel #2
0
func ldbDropFolder(db *leveldb.DB, folder []byte) {
	runtime.GC()

	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	defer snap.Release()

	// Remove all items related to the given folder from the device->file bucket
	start := []byte{keyTypeDevice}
	limit := []byte{keyTypeDevice + 1}
	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	for dbi.Next() {
		itemFolder := deviceKeyFolder(dbi.Key())
		if bytes.Compare(folder, itemFolder) == 0 {
			db.Delete(dbi.Key(), nil)
		}
	}
	dbi.Release()

	// Remove all items related to the given folder from the global bucket
	start = []byte{keyTypeGlobal}
	limit = []byte{keyTypeGlobal + 1}
	dbi = snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	for dbi.Next() {
		itemFolder := globalKeyFolder(dbi.Key())
		if bytes.Compare(folder, itemFolder) == 0 {
			db.Delete(dbi.Key(), nil)
		}
	}
	dbi.Release()
}
Beispiel #3
0
func ldbDropFolder(db *leveldb.DB, folder []byte) {
	runtime.GC()

	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	l.Debugf("created snapshot %p", snap)
	defer func() {
		l.Debugf("close snapshot %p", snap)
		snap.Release()
	}()

	// Remove all items related to the given folder from the device->file bucket
	dbi := snap.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
	for dbi.Next() {
		itemFolder := deviceKeyFolder(dbi.Key())
		if bytes.Compare(folder, itemFolder) == 0 {
			db.Delete(dbi.Key(), nil)
		}
	}
	dbi.Release()

	// Remove all items related to the given folder from the global bucket
	dbi = snap.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
	for dbi.Next() {
		itemFolder := globalKeyFolder(dbi.Key())
		if bytes.Compare(folder, itemFolder) == 0 {
			db.Delete(dbi.Key(), nil)
		}
	}
	dbi.Release()
}
Beispiel #4
0
func InjectLevelDB(jsEngine *JSEngine, db *leveldb.DB) {
	jsEngine.Run("var db = {};")
	dbValue, _ := jsEngine.Get("db")
	dbObj := dbValue.Object()
	dbObj.Set("put", func(call otto.FunctionCall) otto.Value {
		key, err := call.Argument(0).ToString()
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		value, err := call.Argument(1).ToString()
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}

		err = db.Put([]byte(key), []byte(value), nil)
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		return otto.TrueValue()
	})
	dbObj.Set("get", func(call otto.FunctionCall) otto.Value {
		key, err := call.Argument(0).ToString()
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		data, err := db.Get([]byte(key), nil)
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		v, _ := otto.ToValue(string(data))
		return v
	})
	dbObj.Set("remove", func(call otto.FunctionCall) otto.Value {
		key, err := call.Argument(0).ToString()
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		err = db.Delete([]byte(key), nil)
		if err != nil {
			log.Println("Error:", err.Error())
			return otto.FalseValue()
		}
		return otto.TrueValue()
	})
}
Beispiel #5
0
func doOperation(fileNm []byte, finalByt []byte, cmdTyp string, fileDB *leveldb.DB, wo *opt.WriteOptions) {

	var err error

	mutex.Lock()
	switch cmdTyp {
	case "write":
		err = fileDB.Put(fileNm, finalByt, nil)
	case "delete":
		err = fileDB.Delete([]byte(fileNm), wo)
	}
	mutex.Unlock()
	checkError(err)
}
Beispiel #6
0
// Command dele
func delete(conn net.Conn, input_bytes []byte, datadb *leveldb.DB, metadatadb *leveldb.DB) {
	input_string := string(input_bytes)
	inputs := strings.Fields(input_string)
	filename := inputs[1]
	mutex.Lock()
	_, _, _, _, err := read_metadata(filename, metadatadb)

	if err != nil {
		mutex.Unlock()
		conn.Write(append([]byte("ERR_FILE_NOT_FOUND\r\n")))
	} else {

		datadb.Delete([]byte(filename), nil)
		metadatadb.Delete([]byte(filename), nil)
		mutex.Unlock()
		conn.Write(append([]byte("OK\r\n")))

	}
}
Beispiel #7
0
func clean(statsLog io.Writer, db *leveldb.DB) {
	for {
		now := next(cacheLimitSeconds)
		nowSecs := now.Unix()

		var kept, deleted int64
		iter := db.NewIterator(nil, nil)
		for iter.Next() {
			var addrs addressList
			addrs.UnmarshalXDR(iter.Value())

			// Remove expired addresses
			newAddrs := addrs.addresses
			for i := 0; i < len(newAddrs); i++ {
				if nowSecs-newAddrs[i].seen > cacheLimitSeconds {
					newAddrs[i] = newAddrs[len(newAddrs)-1]
					newAddrs = newAddrs[:len(newAddrs)-1]
				}
			}

			// Delete empty records
			if len(newAddrs) == 0 {
				db.Delete(iter.Key(), nil)
				deleted++
				continue
			}

			// Update changed records
			if len(newAddrs) != len(addrs.addresses) {
				addrs.addresses = newAddrs
				db.Put(iter.Key(), addrs.MarshalXDR(), nil)
			}
			kept++
		}
		iter.Release()

		fmt.Fprintf(statsLog, "%d Kept:%d Deleted:%d Took:%0.04fs\n", nowSecs, kept, deleted, time.Since(now).Seconds())
	}
}
Beispiel #8
0
func handleConnection(conn net.Conn, db *leveldb.DB, files *leveldb.DB, expiry *leveldb.DB, mutex *sync.RWMutex) {

	defer conn.Close() // make sure to close the connection even if we panic.
	// Make a buffer to hold incoming data with size bufferSize
	var bufferSize int64 = 1024
	buf := make([]byte, bufferSize)
	var fileName string
	var sLarge string = "" // Initial command string and number of bytes read with the command
	var bufferRead int = 0

	// Iterate over different commands, takes residual string as input
OUTER:
	for {

		// Check residual string from previous command
		buf = []byte(sLarge)
		bufferRead = len([]byte(sLarge))

		// Read the incoming connection into the buffer
		// Warning : We might not get the complete message
		for {
			if strings.Contains(sLarge, "\r\n") || len(sLarge) > 100000 {
				break
			}
			buf = make([]byte, bufferSize)
			n, _ := conn.Read(buf)
			sLarge += string(buf[:n]) // Read till we get the complete command
			bufferRead = n
		}

		//fmt.Println(sLarge, len(sLarge))
		s := strings.Split(sLarge, "\r\n") // Isolate main command from input
		commands := strings.Split(s[0], " ")
		// Remove extra input apart from basic command string
		var remainder int64 = int64(len([]byte(sLarge)) - len([]byte(s[0]+"\r\n")))
		sLarge = sLarge[len(s[0]+"\r\n"):]

		if len(commands) < 2 {
			conn.Write([]byte("ERR_CMD_ERR\r\n"))
			continue OUTER
		} else {
			fileName = commands[1]
		}

		if commands[0] == "read" {
			mutex.RLock()
			file, err := files.Get([]byte(fileName), nil)
			if err != nil {
				conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
				mutex.RUnlock()
				continue OUTER
			}
			//Check expiry
			var diff int64 = 0
			expbytes, err := expiry.Get([]byte(fileName), nil)
			if err == nil {
				exp, _ := strconv.ParseInt(string(expbytes), 10, 64)
				if exp != -1 {
					diff = exp - time.Now().Unix()
				}
			}
			if diff < 0 {
				conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
				mutex.RUnlock()
				continue OUTER
			}

			// Check version
			var version int64 = 0
			data, err := db.Get([]byte(fileName), nil)
			if err == nil {
				version, err = strconv.ParseInt(string(data), 10, 64)
			}
			mutex.RUnlock()

			conn.Write([]byte("CONTENTS " + strconv.FormatInt(version, 10) + " " + strconv.Itoa(len(file)-2) + " " + strconv.FormatInt(diff, 10) + "\r\n"))

			for i := 0; i < len(file)-2; i++ {
				_, err = conn.Write(file[i : i+1])
			}
			_, err = conn.Write([]byte("\r\n"))

			if err != nil && err != io.EOF {
				conn.Write([]byte("ERR_INTERNAL\r\n"))
			}
		} else if commands[0] == "delete" {
			err := files.Delete([]byte(fileName), nil) // Remove entry from database
			//err := os.Remove(fileName)
			if err != nil {
				conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
			} else {
				db.Delete([]byte(fileName), nil)     // Remove entry from database
				expiry.Delete([]byte(fileName), nil) // Remove entry from database
				conn.Write([]byte("OK\r\n"))
			}
		} else if commands[0] == "write" {
			sLarge = ""

			if len(commands) < 3 {
				conn.Write([]byte("ERR_CMD_ERR\r\n"))
				continue OUTER
			}

			// NUmber of bytes to be written to the file
			fileSize, err := strconv.ParseInt(commands[2], 10, 64)
			if err != nil {
				conn.Write([]byte("ERR_CMD_ERR\r\n"))
				continue OUTER
			}
			fileSize = fileSize + int64(len([]byte("\r\n")))

			file := write(conn, fileSize, remainder, bufferRead, buf, &sLarge)

			mutex.Lock()

			err = files.Put([]byte(fileName), file, nil)
			// Get file version
			var version int64 = 0
			data, err := db.Get([]byte(fileName), nil)
			if err == nil {
				version, err = strconv.ParseInt(string(data), 10, 64)
			}

			err = db.Put([]byte(fileName), []byte(strconv.FormatInt(version+1, 10)), nil)
			conn.Write([]byte("OK " + strconv.FormatInt(version+1, 10) + "\r\n"))

			// Write expiry time
			var exp int64 = 0
			if len(commands) > 3 {
				exp, _ = strconv.ParseInt(commands[3], 10, 64)
			}
			if exp == 0 {
				err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(-1, 10)), nil)
			} else {
				err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(time.Now().Unix()+exp, 10)), nil)
			}

			mutex.Unlock()

		} else if commands[0] == "cas" {
			sLarge = ""

			if len(commands) < 4 {
				conn.Write([]byte("ERR_CMD_ERR\r\n"))
				continue OUTER
			}

			// Number of bytes to be written to file
			fileSize, err := strconv.ParseInt(commands[3], 10, 64)
			if err != nil {
				conn.Write([]byte("ERR_CMD_ERR\r\n"))
				continue OUTER
			}
			fileSize = fileSize + int64(len([]byte("\r\n")))

			file := write(conn, fileSize, remainder, bufferRead, buf, &sLarge)

			mutex.Lock()

			// Get file version
			var version int64 = 0
			data, err := db.Get([]byte(fileName), nil)
			if err == nil {
				version, err = strconv.ParseInt(string(data), 10, 64)
			}

			if err != nil {
				conn.Write([]byte("ERR_INTERNAL\r\n"))
				mutex.Unlock()
				return
			}

			fileVersion, err := strconv.ParseInt(commands[2], 10, 64)

			// Check for version match
			if version != fileVersion {
				conn.Write([]byte("ERR_VERSION " + strconv.FormatInt(version, 10) + "\r\n"))
				mutex.Unlock()
				continue OUTER
			}

			err = files.Put([]byte(fileName), file, nil)

			err = db.Put([]byte(fileName), []byte(strconv.FormatInt(version, 10)), nil)
			conn.Write([]byte("OK " + strconv.FormatInt(version, 10) + "\r\n"))

			// Write expiry time
			var exp int64 = 0
			if len(commands) > 4 {
				exp, _ = strconv.ParseInt(commands[4], 10, 64)
			}
			if exp == 0 {
				err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(-1, 10)), nil)
			} else {
				err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(time.Now().Unix()+exp, 10)), nil)
			}

			mutex.Unlock()

		} else {
			conn.Write([]byte("ERR_CMD_ERR\r\n"))
			continue OUTER
		}

	}
}
func levelDbDelete(db *leveldb.DB, key uint64) error {
	bytes := make([]byte, 8)
	util.Uint64toBytes(bytes, key)
	return db.Delete(bytes, nil)
}