func cacheLookup(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) {
var container []map[string]string
for _, each := range way.NodeIDs {
stringid := strconv.FormatInt(each, 10)
data, err := db.Get([]byte(stringid), nil)
if err != nil {
log.Println("denormalize failed for way:", way.ID, "node not found:", stringid)
return container, err
}
s := string(data)
spl := strings.Split(s, ":")
latlon := make(map[string]string)
lat, lon := spl[0], spl[1]
latlon["lat"] = lat
latlon["lon"] = lon
container = append(container, latlon)
}
return container, nil
// fmt.Println(way.NodeIDs)
// fmt.Println(container)
// os.Exit(1)
}
// Command read
func read(conn net.Conn, input_bytes []byte, datadb *leveldb.DB, metadatadb *leveldb.DB) {
input_string := string(input_bytes)
inputs := strings.Fields(input_string)
filename := inputs[1]
mutex.RLock()
version, numbytes, exptime, exp, err1 := read_metadata(filename, metadatadb)
if err1 == nil {
if time.Now().After(exptime) {
mutex.RUnlock()
conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n")) // content has expired
} else {
data, err2 := datadb.Get([]byte(filename), nil)
mutex.RUnlock()
if err2 != nil {
log.Println("error in conversion: ", err1)
}
version_str := strconv.Itoa(version)
numbytes_str := strconv.Itoa(numbytes)
exp_str := "0"
if exp != 0 {
exp_str = strconv.Itoa(int(exptime.Sub(time.Now()).Seconds()))
}
response := append([]byte("CONTENTS "+version_str+" "+numbytes_str+" "+exp_str+" \r\n"), data...)
// log.Println(response)
conn.Write(response)
// log.Println(response)
}
} else {
mutex.RUnlock()
conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
}
}
func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
k := globalKey(folder, file)
bs, err := db.Get(k, nil)
if err == leveldb.ErrNotFound {
return nil
}
if err != nil {
panic(err)
}
var vl versionList
err = vl.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
var devices []protocol.DeviceID
for _, v := range vl.versions {
if !v.version.Equal(vl.versions[0].version) {
break
}
n := protocol.DeviceIDFromBytes(v.device)
devices = append(devices, n)
}
return devices
}
func ldbAvailability(db *leveldb.DB, repo, file []byte) []protocol.NodeID {
k := globalKey(repo, file)
bs, err := db.Get(k, nil)
if err == leveldb.ErrNotFound {
return nil
}
if err != nil {
panic(err)
}
var vl versionList
err = vl.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
var nodes []protocol.NodeID
for _, v := range vl.versions {
if v.version != vl.versions[0].version {
break
}
var n protocol.NodeID
copy(n[:], v.node)
nodes = append(nodes, n)
}
return nodes
}
/*
/Read the DATABASE
/Read the Key and Value by Key
*/
func dbRead(dbObj *leveldb.DB, readDBKey []byte) []byte {
data, readError := dbObj.Get([]byte(readDBKey), nil)
if readError != nil {
fmt.Println("DB Read ERROR")
}
return data
}
func read_metadata(filename string, metadatadb *leveldb.DB) (version int, numbytes int, exptime time.Time, exp int, err error) {
data, err2 := metadatadb.Get([]byte(filename), nil)
// log.Println(data, filename, err2)
err = err2
if err != nil {
version = 0
numbytes = 0
exptime = time.Now()
exp = 0
} else {
line := string(data)
fields := strings.Fields(line)
version, _ = strconv.Atoi(fields[0])
numbytes, _ = strconv.Atoi(fields[1])
if len(fields) == 7 {
exptime_str := fields[2] + " " + fields[3] + " " + fields[4] + " " + fields[5]
layout := "2006-01-02 15:04:05 -0700 MST"
exptime, _ = time.Parse(layout, exptime_str)
exp, _ = strconv.Atoi(fields[6])
} else {
log.Println("why here?", string(data), string(filename))
}
}
return
}
func BitIndexes(bdb *leveldb.DB, bmd5s []string) map[uint]struct{} {
wg := new(sync.WaitGroup)
res := make(map[uint]struct{})
bitChan := make(chan uint, 1000)
go func() {
for bit := range bitChan {
if _, ok := res[bit]; !ok {
res[bit] = struct{}{}
}
}
}()
for _, bmd5 := range bmd5s {
wg.Add(1)
go func(bmd5 string) {
var fpmap Fpmap
data, _ := bdb.Get([]byte("m-"+bmd5), nil)
if err := json.Unmarshal(data, &fpmap); err != nil {
Error.Println("Error decoding fpmap object", bmd5)
}
for _, index := range fpmap {
bitChan <- index.bindex
}
}(bmd5)
}
wg.Wait()
close(bitChan)
return res
}
func get(db *leveldb.DB, id protocol.DeviceID) []address {
var addrs addressList
val, err := db.Get(id[:], nil)
if err == nil {
addrs.UnmarshalXDR(val)
}
return addrs.addresses
}
func getVersion(db *leveldb.DB) (int64, error) {
data, err := db.Get([]byte(versionKey), nil)
if err == leveldb.ErrNotFound {
return nilDataVersion, nil
} else if len(data) != 8 {
return 0, fmt.Errorf("version value format is unknown")
}
return int64(order.Uint64(data)), nil
}
func saveVideos(db *leveldb.DB, videos Videos) {
for _, video := range videos {
data, _ := db.Get([]byte(video.Url), nil)
if len(data) == 0 {
encoded, _ := json.Marshal(video)
_ = db.Put([]byte(video.Url), []byte(encoded), nil)
}
}
}
func InjectLevelDB(jsEngine *JSEngine, db *leveldb.DB) {
jsEngine.Run("var db = {};")
dbValue, _ := jsEngine.Get("db")
dbObj := dbValue.Object()
dbObj.Set("put", func(call otto.FunctionCall) otto.Value {
key, err := call.Argument(0).ToString()
if err != nil {
log.Println("Error:", err.Error())
return otto.FalseValue()
}
value, err := call.Argument(1).ToString()
if err != nil {
log.Println("Error:", err.Error())
return otto.FalseValue()
}
err = db.Put([]byte(key), []byte(value), nil)
if err != nil {
log.Println("Error:", err.Error())
return otto.FalseValue()
}
return otto.TrueValue()
})
dbObj.Set("get", func(call otto.FunctionCall) otto.Value {
key, err := call.Argument(0).ToString()
if err != nil {
log.Println("Error:", err.Error())
return otto.FalseValue()
}
data, err := db.Get([]byte(key), nil)
if err != nil {
log.Println("Error:", err.Error())
return otto.FalseValue()
}
v, _ := otto.ToValue(string(data))
return v
})
dbObj.Set("remove", func(call otto.FunctionCall) otto.Value {
key, err := call.Argument(0).ToString()
if err != nil {
log.Println("Error:", err.Error())
return otto.FalseValue()
}
err = db.Delete([]byte(key), nil)
if err != nil {
log.Println("Error:", err.Error())
return otto.FalseValue()
}
return otto.TrueValue()
})
}
func doRead(cmd *utils.Cmd, db *leveldb.DB) *utils.Cmd {
dirMutex.RLock()
defer dirMutex.RUnlock()
fi := dir[cmd.Filename]
//if fi != nil {
// // exptime := 0
// if fi.Timer != nil {
// // exptime := fi.Exptime.Sub(time.Now())
// // if exptime < 0 {
// // exptime = 0
// // }
// }
//} else {
//
//}
//data, err := db.Get([]byte(cmd.Filename), nil)
//if err != nil {
// log.Fatal(err)
//}
//response := fmt.Sprintf("CONTENTS %d %d %d\r\n", fi.Version, fi.Numbytes, fi.Exptime)
//con.Write([]byte(response))
//con.Write(data)
//con.Write([]byte("\r\n"))
if fi != nil {
var remainingTime int64
remainingTime = 0
if fi.Timer != nil {
remainingTime := int(fi.Exptime.Sub(time.Now()))
if remainingTime < 0 {
remainingTime = 0
}
}
data, err := db.Get([]byte(cmd.Filename), nil)
if err != nil {
log.Fatal(err)
}
return &utils.Cmd{
Type: "C",
Filename: cmd.Filename,
Content: data,
Numbytes: fi.Numbytes,
Exptime: remainingTime,
Version: fi.Version,
}
} else {
return &utils.Cmd{Type: "F"} // file not found
}
}
func ldbGet(db *leveldb.DB, repo, node, file []byte) protocol.FileInfo {
nk := nodeKey(repo, node, file)
bs, err := db.Get(nk, nil)
if err == leveldb.ErrNotFound {
return protocol.FileInfo{}
}
if err != nil {
panic(err)
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
return f
}
func ldbGet(db *leveldb.DB, folder, device, file []byte) (protocol.FileInfo, bool) {
nk := deviceKey(folder, device, file)
bs, err := db.Get(nk, nil)
if err == leveldb.ErrNotFound {
return protocol.FileInfo{}, false
}
if err != nil {
panic(err)
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
return f, true
}
func Worker(benDB, malDB *leveldb.DB, keysChan chan *KeyPair, resChan chan string, wg *sync.WaitGroup) {
// Decreasing internal counter for wait-group as soon as goroutine finishes
defer wg.Done()
var score float64
var bhash, mhash []byte
for keyPair := range keysChan {
// Todo: meta info check
if MetaCheck(benDB, malDB, keyPair.bmd5, keyPair.mmd5) {
bhash, _ = benDB.Get(keyPair.bkey, nil)
mhash, _ = malDB.Get(keyPair.mkey, nil)
if len(bhash) > 0 && len(mhash) > 0 {
score = HashCompare(bhash, mhash, keyPair.bsize, keyPair.msize)
resChan <- fmt.Sprintf("%v, %v, %.3f\n", keyPair.bmd5, keyPair.mmd5, score)
}
}
}
}
func Worker(indir, oridir, diffdir string, mdb *leveldb.DB, inChan chan InputPair, wg *sync.WaitGroup) {
// Decreasing internal counter for wait-group as soon as goroutine finishes
defer wg.Done()
for inPair := range inChan {
var fpmap Fpmap
var indexList []Index
data, _ := mdb.Get([]byte("m-"+inPair.mmd5), nil)
if err := json.Unmarshal(data, &fpmap); err != nil {
Error.Println("Error decoding fpmap object", inPair.mmd5)
}
for _, index := range fpmap {
if _, ok := inPair.bbits[index.bindex]; !ok {
indexList = append(indexList, index)
}
}
indexMap := CombineIndexes(&indexList)
go GetDiff(inPair.mmd5, indir, oridir, diffdir, indexMap)
}
}
func update(db *leveldb.DB, id protocol.DeviceID, addrs []address) {
var newAddrs addressList
val, err := db.Get(id[:], nil)
if err == nil {
newAddrs.UnmarshalXDR(val)
}
nextAddr:
for _, newAddr := range addrs {
for i, exAddr := range newAddrs.addresses {
if bytes.Compare(newAddr.ip, exAddr.ip) == 0 {
newAddrs.addresses[i] = newAddr
continue nextAddr
}
}
newAddrs.addresses = append(newAddrs.addresses, newAddr)
}
db.Put(id[:], newAddrs.MarshalXDR(), nil)
}
func fetchZip(db *leveldb.DB, zip string) (*zipi.Point, error) {
//geocoder := new(zip.Point)
geo, err := db.Get([]byte(zip), nil)
if err != nil {
return nil, errors.New(fmt.Sprintf("zip: %v not found", zip))
}
fmt.Printf("Geo Byte Array: %v", geo)
fmt.Printf("Geo %v+", string(geo))
//buffer := bytes.NewReader(geo)
//err = binary.Read(buffer, binary.LittleEndian, &geocoder)
//if err != nil {
//fmt.Println("binary.Read failed:", err)
//}
part := strings.Split(string(geo), ":")
x, _ := strconv.ParseFloat(part[0], 64)
y, _ := strconv.ParseFloat(part[1], 64)
geocoder := new(zipi.Point)
geocoder.X = x
geocoder.Y = y
fmt.Printf("Geo Coder: x= %v, y=%v", geocoder.X, geocoder.Y)
return geocoder, nil
}
func handleConnection(conn net.Conn, db *leveldb.DB, files *leveldb.DB, expiry *leveldb.DB, mutex *sync.RWMutex) {
defer conn.Close() // make sure to close the connection even if we panic.
// Make a buffer to hold incoming data with size bufferSize
var bufferSize int64 = 1024
buf := make([]byte, bufferSize)
var fileName string
var sLarge string = "" // Initial command string and number of bytes read with the command
var bufferRead int = 0
// Iterate over different commands, takes residual string as input
OUTER:
for {
// Check residual string from previous command
buf = []byte(sLarge)
bufferRead = len([]byte(sLarge))
// Read the incoming connection into the buffer
// Warning : We might not get the complete message
for {
if strings.Contains(sLarge, "\r\n") || len(sLarge) > 100000 {
break
}
buf = make([]byte, bufferSize)
n, _ := conn.Read(buf)
sLarge += string(buf[:n]) // Read till we get the complete command
bufferRead = n
}
//fmt.Println(sLarge, len(sLarge))
s := strings.Split(sLarge, "\r\n") // Isolate main command from input
commands := strings.Split(s[0], " ")
// Remove extra input apart from basic command string
var remainder int64 = int64(len([]byte(sLarge)) - len([]byte(s[0]+"\r\n")))
sLarge = sLarge[len(s[0]+"\r\n"):]
if len(commands) < 2 {
conn.Write([]byte("ERR_CMD_ERR\r\n"))
continue OUTER
} else {
fileName = commands[1]
}
if commands[0] == "read" {
mutex.RLock()
file, err := files.Get([]byte(fileName), nil)
if err != nil {
conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
mutex.RUnlock()
continue OUTER
}
//Check expiry
var diff int64 = 0
expbytes, err := expiry.Get([]byte(fileName), nil)
if err == nil {
exp, _ := strconv.ParseInt(string(expbytes), 10, 64)
if exp != -1 {
diff = exp - time.Now().Unix()
}
}
if diff < 0 {
conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
mutex.RUnlock()
continue OUTER
}
// Check version
var version int64 = 0
data, err := db.Get([]byte(fileName), nil)
if err == nil {
version, err = strconv.ParseInt(string(data), 10, 64)
}
mutex.RUnlock()
conn.Write([]byte("CONTENTS " + strconv.FormatInt(version, 10) + " " + strconv.Itoa(len(file)-2) + " " + strconv.FormatInt(diff, 10) + "\r\n"))
for i := 0; i < len(file)-2; i++ {
_, err = conn.Write(file[i : i+1])
}
_, err = conn.Write([]byte("\r\n"))
if err != nil && err != io.EOF {
conn.Write([]byte("ERR_INTERNAL\r\n"))
}
} else if commands[0] == "delete" {
err := files.Delete([]byte(fileName), nil) // Remove entry from database
//err := os.Remove(fileName)
if err != nil {
conn.Write([]byte("ERR_FILE_NOT_FOUND\r\n"))
} else {
db.Delete([]byte(fileName), nil) // Remove entry from database
expiry.Delete([]byte(fileName), nil) // Remove entry from database
conn.Write([]byte("OK\r\n"))
}
} else if commands[0] == "write" {
sLarge = ""
if len(commands) < 3 {
conn.Write([]byte("ERR_CMD_ERR\r\n"))
continue OUTER
}
// NUmber of bytes to be written to the file
fileSize, err := strconv.ParseInt(commands[2], 10, 64)
if err != nil {
conn.Write([]byte("ERR_CMD_ERR\r\n"))
continue OUTER
}
fileSize = fileSize + int64(len([]byte("\r\n")))
file := write(conn, fileSize, remainder, bufferRead, buf, &sLarge)
mutex.Lock()
err = files.Put([]byte(fileName), file, nil)
// Get file version
var version int64 = 0
data, err := db.Get([]byte(fileName), nil)
if err == nil {
version, err = strconv.ParseInt(string(data), 10, 64)
}
err = db.Put([]byte(fileName), []byte(strconv.FormatInt(version+1, 10)), nil)
conn.Write([]byte("OK " + strconv.FormatInt(version+1, 10) + "\r\n"))
// Write expiry time
var exp int64 = 0
if len(commands) > 3 {
exp, _ = strconv.ParseInt(commands[3], 10, 64)
}
if exp == 0 {
err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(-1, 10)), nil)
} else {
err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(time.Now().Unix()+exp, 10)), nil)
}
mutex.Unlock()
} else if commands[0] == "cas" {
sLarge = ""
if len(commands) < 4 {
conn.Write([]byte("ERR_CMD_ERR\r\n"))
continue OUTER
}
// Number of bytes to be written to file
fileSize, err := strconv.ParseInt(commands[3], 10, 64)
if err != nil {
conn.Write([]byte("ERR_CMD_ERR\r\n"))
continue OUTER
}
fileSize = fileSize + int64(len([]byte("\r\n")))
file := write(conn, fileSize, remainder, bufferRead, buf, &sLarge)
mutex.Lock()
// Get file version
var version int64 = 0
data, err := db.Get([]byte(fileName), nil)
if err == nil {
version, err = strconv.ParseInt(string(data), 10, 64)
}
if err != nil {
conn.Write([]byte("ERR_INTERNAL\r\n"))
mutex.Unlock()
return
}
fileVersion, err := strconv.ParseInt(commands[2], 10, 64)
// Check for version match
if version != fileVersion {
conn.Write([]byte("ERR_VERSION " + strconv.FormatInt(version, 10) + "\r\n"))
mutex.Unlock()
continue OUTER
}
err = files.Put([]byte(fileName), file, nil)
err = db.Put([]byte(fileName), []byte(strconv.FormatInt(version, 10)), nil)
conn.Write([]byte("OK " + strconv.FormatInt(version, 10) + "\r\n"))
// Write expiry time
var exp int64 = 0
if len(commands) > 4 {
exp, _ = strconv.ParseInt(commands[4], 10, 64)
}
if exp == 0 {
err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(-1, 10)), nil)
} else {
err = expiry.Put([]byte(fileName), []byte(strconv.FormatInt(time.Now().Unix()+exp, 10)), nil)
}
mutex.Unlock()
} else {
conn.Write([]byte("ERR_CMD_ERR\r\n"))
continue OUTER
}
}
}
