func (ls *Libstore) iPut(key, value string) error {
indicator := ls.getServerIndicator(key)
con, e := ls.getServerConnection(indicator)
if e != nil {
return e
}
args := &storageproto.PutArgs{key, value}
var reply storageproto.PutReply
lsplog.Vlogf(1, "Calling RPC StorageRPC.Put with key %s and value %s", key, value)
e = con.Call("StorageRPC.Put", args, &reply)
if e != nil {
return e
}
lsplog.Vlogf(1, "Called RPC StorageRPC.Put successfully")
lsplog.Vlogf(1, "Status recieved %v", reply.Status)
if reply.Status == storageproto.EWRONGSERVER {
lsplog.Vlogf(1, "Key %s should not be called from server", key)
return lsplog.MakeErr("Wrong server is called")
} else if reply.Status == storageproto.EPUTFAILED {
lsplog.Vlogf(1, "Error happened while putting")
return lsplog.MakeErr("Error while putting error")
} else if reply.Status != storageproto.OK {
lsplog.Vlogf(1, "Error happened while putting")
return lsplog.MakeErr("Error while putting error")
}
return nil
}
func (ls *Libstore) iAppendToList(key, newitem string) error {
indicator := ls.getServerIndicator(key)
con, e := ls.getServerConnection(indicator)
if e != nil {
return e
}
args := &storageproto.PutArgs{key, newitem}
var reply storageproto.PutReply
lsplog.Vlogf(1, "Calling RPC StorageRPC.AppendToList with key %s and value %s", key, newitem)
e = con.Call("StorageRPC.AppendToList", args, &reply)
if e != nil {
return e
}
lsplog.Vlogf(1, "Called RPC StorageRPC.AppendToList successfully")
lsplog.Vlogf(1, "Status recieved %v", reply.Status)
if reply.Status == storageproto.EWRONGSERVER {
lsplog.Vlogf(1, "Key %s should not be called from server", key)
return lsplog.MakeErr("Wrong server is called")
} else if reply.Status == storageproto.EITEMNOTFOUND {
lsplog.Vlogf(1, "Item to add not found in server")
return lsplog.MakeErr("Error: item to add not found")
} else if reply.Status == storageproto.EITEMEXISTS {
lsplog.Vlogf(1, "Item to add alrady exists")
return lsplog.MakeErr("Error: item to add exists")
}
return nil
}
func (ls *Libstore) getServers(masterConnection *rpc.Client) error {
lsplog.Vlogf(1, "Getting nodes from masters")
args := new(storageproto.GetServersArgs)
var reply storageproto.RegisterReply
e := masterConnection.Call("StorageRPC.GetServers", &args, &reply)
if e != nil {
return e
}
numberOfTries := 0
for !reply.Ready {
time.Sleep(WAIT_FOR_STORAGE_SLEEP * time.Second)
if numberOfTries < RETRY_THRESH {
e := masterConnection.Call("StorageRPC.GetServers", &args, &reply)
if e != nil {
lsplog.Vlogf(3, "Connection to master failed")
return e
}
numberOfTries++
} else {
return lsplog.MakeErr("Waited too long for storage server to be ready")
}
}
lsplog.Vlogf(1, "servers %+v", reply.Servers)
ls.servers = reply.Servers
return nil
}
func (ss *Storageserver) GetServers(args *storageproto.GetServersArgs,
reply *storageproto.RegisterReply) error {
fmt.Println("Storage GetServers invoked")
if !ss.isMaster {
fmt.Println("WARNING:Calling a non-master node for GetServers")
return lsplog.MakeErr("Calling a non-master node to GetServers")
}
if len(ss.nodes) != ss.numnodes {
fmt.Println("GetServer not ready")
//what a hack here, need change if time possible
time.Sleep(time.Duration(2*1000) * time.Millisecond)
reply.Ready = false
return nil
}
servers := make([]storageproto.Node, ss.numnodes)
i := 0
for node, _ := range ss.nodes {
//fmt.Printf("i: %d, info: %v\n", i, node)
servers[i] = node
i++
}
reply.Servers = servers
reply.Ready = true
return nil
}
/**@brief helper function for sorting
* @param server master storage server addr
* @param myhostport trib server's port
* @param flags
* @return *Libstore
* @return error
*/
func iNewLibstore(server, myhostport string, flags int) (*Libstore, error) {
var store Libstore
var master *rpc.Client
var args storageproto.GetServersArgs
var reply storageproto.RegisterReply
var err error
store.Addr = myhostport
store.Flags = flags
if store.Addr != "" {
rpc.Register(cacherpc.NewCacheRPC(&store))
}
lsplog.Vlogf(3, "libstore try to connect to master storage %s", server)
master, err = rpc.DialHTTP("tcp", server)
if lsplog.CheckReport(1, err) {
return nil, err
}
lsplog.Vlogf(3, "try to call GetServers")
master.Call("StorageRPC.GetServers", &args, &reply)
if !reply.Ready {
for i := 0; i < 5; i++ {
time.Sleep(1000 * time.Millisecond)
master.Call("StorageRPC.GetServers", &args, &reply)
}
}
err = master.Close()
if lsplog.CheckReport(1, err) {
lsplog.Vlogf(3, "WARNING close master failed")
}
// couldn't get list of servers from master
if (reply.Ready == false) || (reply.Servers == nil) {
return nil, lsplog.MakeErr("Storage system not ready.")
}
store.Nodes = reply.Servers
store.RPCConn = make([]*rpc.Client, len(store.Nodes))
sort.Sort(store.Nodes)
/*
for i := 0; i < len(store.Nodes); i++ {
fmt.Printf("%v\n", store.Nodes[i])
}*/
store.Leases = cache.NewCache()
if lsplog.CheckReport(1, err) {
return nil, err
}
lsplog.Vlogf(3, "libstore create complete")
return &store, nil
}
/**@brief Get is the most important function for cache, it will first clear all
* the expire entries, and then fetch the cache content or add the count
* of wantlease flag
* @param key
* @param GetArgs
* @return interface{}(string or []string)
* @return error
*/
func (cache *Cache) Get(
key string, args *storageproto.GetArgs) (interface{}, error) {
var entry *Entry
var valid bool
var data interface{}
//fmt.Printf("Cache get: %s\n", key)
cache.Lock.Lock()
cache.ClearExpired()
entry, valid = cache.Map[key]
if !valid {
entry = new(Entry)
entry.Queries = list.New()
entry.Queries.PushBack(time.Now())
//fmt.Printf("Cache entry: %+v\n", *entry)
//fmt.Printf("Queries: %+v\n", entry.Queries)
cache.Map[key] = entry
cache.Lock.Unlock()
return "", lsplog.MakeErr("Not found.")
}
if entry.Granted {
data = entry.Data
cache.Lock.Unlock()
//fmt.Printf("Already in Cache %s->%v\n", key, data)
return data, nil
}
entry.Queries.PushBack(time.Now())
//fmt.Printf("Cache entry: %v\n", *entry)
if entry.Queries.Len() > storageproto.QUERY_CACHE_THRESH {
//fmt.Printf("QUERY_CACHE_THRESH reached. Asking for lease.\n")
args.WantLease = true
}
cache.Lock.Unlock()
return "", lsplog.MakeErr("Not in cache")
}
func (ls *Libstore) iGet(key string) (string, error) {
cached := ls.getFromCache(key)
if cached != nil {
return cached[0], nil
}
indicator := ls.getServerIndicator(key)
con, e := ls.getServerConnection(indicator)
if e != nil {
return "", e
}
args := &storageproto.GetArgs{Key: key}
ls.checkWhetherLeaseNeeded(args, key)
var reply storageproto.GetReply
lsplog.Vlogf(1, "[Get] Calling RPC StorageRPC.Get on server")
e = con.Call("StorageRPC.Get", args, &reply)
if e != nil {
lsplog.Vlogf(1, "[Get] Calling RPC StorageRPC.Get has an error")
return "", e
}
lsplog.Vlogf(1, "[Get] Called RPC StorageRPC.Get successfully")
lsplog.Vlogf(1, "[Get] Status recieved %v", reply.Status)
if reply.Status == storageproto.EKEYNOTFOUND {
return "", lsplog.MakeErr("Key not found")
} else if reply.Status == storageproto.EWRONGSERVER {
lsplog.Vlogf(1, "[Get] Key %s should not be called from server", key)
return "", lsplog.MakeErr("Wrong server is called")
} else if reply.Status != storageproto.OK {
lsplog.Vlogf(1, "[Get] Error status %v is recieved from server", reply.Status)
return "", lsplog.MakeErr("Unknown error")
}
if reply.Lease.Granted {
if ls.flags == NONE {
ls.requestsLocker.Lock()
delete(ls.requests, key)
ls.requestsLocker.Unlock()
}
ls.cacheLocker.Lock()
lsplog.Vlogf(1, "[Get] Adding key %s to cache", key)
ls.cache[key] = &Leased{time.Now().Add(time.Duration(reply.Lease.ValidSeconds) * time.Second), []string{reply.Value}}
ls.cacheLocker.Unlock()
}
return reply.Value, nil
}
func (ls *Libstore) iAppendToList(key, newitem string) error {
args := &storageproto.PutArgs{key, newitem}
var reply storageproto.PutReply
cli, _ := ls.getServer(key)
err := cli.Call("StorageRPC.AppendToList", args, &reply)
if err != nil {
return err
}
if reply.Status != storageproto.OK {
return lsplog.MakeErr("AppendToList failed: Storage error")
}
return nil
}
func (ss *Storageserver) addLeasePool(args *storageproto.GetArgs,
lease *storageproto.LeaseStruct) error {
fmt.Printf("add key %s to lease pool\n", args.Key)
var tmp = false
var mtx sync.Mutex
entry, present := ss.leasePool[args.Key]
lease.Granted = true
lease.ValidSeconds = storageproto.LEASE_SECONDS
if present {
//make sure do not grant duplicate lease
holder := search(entry.holders, args.LeaseClient)
if holder != nil {
fmt.Printf("holder isTimeout %t, pending %t\n", isTimeout(*holder),
*(entry.pending))
if isTimeout(*holder) || !(*(entry.pending)) {
holder.issueTime = time.Now()
} else {
lease.Granted = false
lease.ValidSeconds = 0
return lsplog.MakeErr("trying to issue duplicate lease")
}
return nil
}
} else {
ss.leasePool[args.Key] = leaseEntry{nil, nil, &mtx}
entry = ss.leasePool[args.Key]
entry.pending = &tmp
}
holder := make([]leaseHolder, 1)
holder[0] = leaseHolder{args.LeaseClient, time.Now()}
entry.holders = append(holder, (entry.holders)...)
(ss.leasePool[args.Key]) = entry
return nil
}
func (ls *Libstore) iPut(key, value string) error {
args := &storageproto.PutArgs{key, value}
var reply storageproto.PutReply
cli, err := ls.getServer(key)
if err != nil {
fmt.Fprintf(os.Stderr, "error in get server\n")
return err
}
err = cli.Call("StorageRPC.Put", args, &reply)
if err != nil {
return err
}
if reply.Status != storageproto.OK {
return lsplog.MakeErr("Put failed: Storage error")
}
return nil
}
// Non-master servers to the master
func (ss *Storageserver) RegisterServer(args *storageproto.RegisterArgs,
reply *storageproto.RegisterReply) error {
fmt.Printf("st registerServer invoked\n")
if !ss.isMaster {
lsplog.Vlogf(0, "WARNING:Calling a non-master node to register")
return lsplog.MakeErr("Calling a non-master node to register")
}
_, present := ss.nodes[args.ServerInfo]
if !present {
//add to nodes
ss.nodes[args.ServerInfo] = true
fmt.Println("add nodes %v", args.ServerInfo)
}
fmt.Printf("master collect slave info %d/%d\n", len(ss.nodes), ss.numnodes)
reply.Servers = nil
if len(ss.nodes) == ss.numnodes {
reply.Ready = true
//ss.GetServers(nil, reply)
servers := make([]storageproto.Node, ss.numnodes)
i := 0
for node, _ := range ss.nodes {
//fmt.Printf("i: %d, info: %v\n", i, node)
servers[i] = node
i++
}
reply.Servers = servers
} else {
reply.Ready = false
}
return nil
}
/**@brief get posted tribbles
* @param GetTribblesArgs
* @param GetTribblesReply
* @return error
*/
func (ts *Tribserver) GetTribbles(
args *tribproto.GetTribblesArgs, reply *tribproto.GetTribblesReply) error {
var trib_key string
var trib_ids []string
var trib_enc string
var err error
var length int
trib_key = fmt.Sprintf("%s:T", args.Userid)
trib_ids, err = ts.Store.GetList(trib_key)
if lsplog.CheckReport(1, err) {
reply.Status = tribproto.ENOSUCHUSER
reply.Tribbles = nil
return nil
}
reply.Status = tribproto.OK
if len(trib_ids) > 100 {
length = 100
} else {
length = len(trib_ids)
}
reply.Tribbles = make([]tribproto.Tribble, length)
for i := 0; i < length; i++ {
trib_enc, err = ts.Store.Get(trib_ids[len(trib_ids)-1-i])
if lsplog.CheckReport(1, err) {
return lsplog.MakeErr("Get Tribbles Message Error")
}
//fmt.Printf("unmarshal string %s\n", trib_enc)
_ = json.Unmarshal([]byte(trib_enc), &(reply.Tribbles[i]))
}
return nil
}
func iNewLibstore(server string, myhostport string, flags int) (*Libstore, error) {
ls := &Libstore{}
ls.flags = flags
ls.myhostport = myhostport
// Create RPC connection to storage server
cli, err := rpc.DialHTTP("tcp", server)
if err != nil {
fmt.Printf("Could not connect to server %s, returning nil\n", server)
return nil, err
}
ls.cli = cli
// Get list of storage servers from master storage server
// will retry five times after waiting for 1 second each try
triesLeft := 5
for triesLeft > 0 {
args := storageproto.GetServersArgs{}
var reply storageproto.RegisterReply
err = cli.Call("StorageRPC.GetServers", args, &reply)
if err != nil {
return nil, err
}
if reply.Ready {
ls.nodelist = reply.Servers
break
}
triesLeft--
time.Sleep(time.Duration(1) * time.Second)
}
if ls.nodelist == nil {
return nil, lsplog.MakeErr("Storage system not ready")
}
sort.Sort(byID{ls.nodelist})
//initialize the connCache
ls.connCache = make(map[string]*rpc.Client)
ls.cacheM = make(chan int, 1)
ls.cacheM <- 1
ls.getCache = make(map[string]string)
ls.getM = make(chan int, 1)
ls.getM <- 1
ls.getListCache = make(map[string]([]string))
ls.getListM = make(chan int, 1)
ls.getListM <- 1
ls.leaseMap = make(map[string]LeaseQuery)
ls.leaseM = make(chan int, 1)
ls.leaseM <- 1
ls.crpc = cacherpc.NewCacheRPC(ls)
rpc.Register(ls.crpc)
// do NOT connect to other storage servers here
// this should be done in a lazy fashion upon the first use
// of the storage server, and then the rpc connection can be cached
//Start up garbage collector
go ls.GarbageCollector()
return ls, nil
}
func (ls *Libstore) iGetList(key string) ([]string, error) {
now := time.Now().UnixNano()
//check if lease is still valid
<-ls.leaseM
//fmt.Println("looking for lease for key: %v", key)
lease, found := ls.leaseMap[key]
ls.leaseM <- 1
if found == true {
if lease.LeaseTimeout > 0 {
if now < lease.LeaseTimeout {
<-ls.getListM
value, gFound := ls.getListCache[key]
ls.getListM <- 1
if gFound == true {
return value, nil
}
}
ls.ClearCaches(key)
}
if now < lease.RequestLeaseTime {
lease.Queries++
} else {
lease.Queries = 1
}
ls.leaseMap[key] = lease
} else {
requestTime := time.Now().Add(storageproto.QUERY_CACHE_SECONDS * time.Second).UnixNano()
ls.leaseMap[key] = LeaseQuery{Queries: 1, RequestLeaseTime: requestTime, LeaseTimeout: 0}
}
var wantlease bool
if ls.myhostport != "" && (ls.flags == ALWAYS_LEASE || lease.Queries >= storageproto.QUERY_CACHE_THRESH) {
wantlease = true
} else {
wantlease = false
}
args := &storageproto.GetArgs{key, wantlease, ls.myhostport}
var reply storageproto.GetListReply
cli, err := ls.getServer(key)
if err != nil {
fmt.Fprintf(os.Stderr, "error in get server\n")
return nil, err
}
err = cli.Call("StorageRPC.GetList", args, &reply)
if err != nil {
return nil, err
}
if reply.Status != storageproto.OK {
return nil, lsplog.MakeErr("GetList failed: Storage error")
}
if reply.Lease.Granted == true {
nano := time.Now().Add(time.Duration(reply.Lease.ValidSeconds) * time.Second).UnixNano()
<-ls.leaseM
ls.leaseMap[key] = LeaseQuery{LeaseTimeout: nano}
ls.leaseM <- 1
<-ls.getListM
ls.getListCache[key] = reply.Value
ls.getListM <- 1
}
return reply.Value, nil
}
/**@brief helper function for sorting
* @param function
* @param status
* @return error
*/
func MakeErr(function string, status int) lsplog.LspErr {
var str string
str = fmt.Sprintf("%s failed: %s (%d)", function, StatusName[status], status)
return lsplog.MakeErr(str)
}