func iNewLibstore(server, myhostport string, flags int) (*Libstore, error) {
ls := new(Libstore)
lsplog.Vlogf(3, "[iNewLibstore] Libstore flag %d", flags)
ls.cache = make(map[string]*Leased)
ls.cacheLocker = new(sync.Mutex)
ls.cacheCallbackHostPort = myhostport
if flags == NONE {
ls.requests = make(map[string]*Request)
ls.requestsLocker = new(sync.Mutex)
}
if myhostport != "" {
go ls.revokeLeasesListenerInit(myhostport)
go ls.cleanCache()
}
ls.flags = flags
masterConnection, e := rpc.DialHTTP("tcp", server)
if e != nil {
lsplog.Vlogf(3, "[iNewLibstore] Error while connecting master node")
return nil, e
}
ls.connections = make(map[string]*rpc.Client)
ls.connections[server] = masterConnection
e = ls.getServers(masterConnection)
if e != nil {
return nil, e
}
return ls, nil
}
func (ls *Libstore) checkWhetherLeaseNeeded(args *storageproto.GetArgs, key string) {
if ls.flags == ALWAYS_LEASE {
args.WantLease = true
args.LeaseClient = ls.cacheCallbackHostPort
} else if ls.cacheCallbackHostPort == "" {
args.WantLease = false
} else {
ls.requestsLocker.Lock()
if ls.requests[key] == nil {
lsplog.Vlogf(3, "[checkWhetherLeaseNeeded] First requested")
req := &Request{1, time.Now().Add(time.Duration(storageproto.QUERY_CACHE_SECONDS) * time.Second)}
ls.requests[key] = req
} else {
req := ls.requests[key]
if time.Now().Before(req.ts) {
req.frequency = req.frequency + 1
if req.frequency >= storageproto.QUERY_CACHE_THRESH {
lsplog.Vlogf(3, "[checkWhetherLeaseNeeded] Going to ask for a lease")
args.LeaseClient = ls.cacheCallbackHostPort
args.WantLease = true
} else {
args.WantLease = false
}
} else {
delete(ls.requests, key)
args.WantLease = false
}
}
ls.requestsLocker.Unlock()
}
}
func (ss *Storageserver) GetList(args *storageproto.GetArgs,
reply *storageproto.GetListReply) error {
lsplog.Vlogf(3, "storage try to getlist with key %s", args.Key)
//ss.rwlock.RLock()
val, present := ss.hash[args.Key]
if !present {
if ss.numnodes == 1 {
reply.Status = storageproto.EKEYNOTFOUND
} else {
reply.Status = storageproto.EWRONGSERVER
}
reply.Value = nil
//ss.rwlock.RUnlock()
return nil
}
lsplog.Vlogf(3, "storage getlist key %s, val %s", args.Key, val)
err := json.Unmarshal([]byte(val), &(reply.Value))
if err != nil {
lsplog.Vlogf(0, "WARNING: unmarshal data generate an error")
}
reply.Status = storageproto.OK
if args.WantLease {
ss.addLeasePool(args, &(reply.Lease))
}
//ss.rwlock.RUnlock()
return nil
}
func (ls *Libstore) getServers(masterConnection *rpc.Client) error {
lsplog.Vlogf(1, "Getting nodes from masters")
args := new(storageproto.GetServersArgs)
var reply storageproto.RegisterReply
e := masterConnection.Call("StorageRPC.GetServers", &args, &reply)
if e != nil {
return e
}
numberOfTries := 0
for !reply.Ready {
time.Sleep(WAIT_FOR_STORAGE_SLEEP * time.Second)
if numberOfTries < RETRY_THRESH {
e := masterConnection.Call("StorageRPC.GetServers", &args, &reply)
if e != nil {
lsplog.Vlogf(3, "Connection to master failed")
return e
}
numberOfTries++
} else {
return lsplog.MakeErr("Waited too long for storage server to be ready")
}
}
lsplog.Vlogf(1, "servers %+v", reply.Servers)
ls.servers = reply.Servers
return nil
}
/**@brief create a new user
* @param CreateUserArgs
* @param CreateUserReply
* @return error
*/
func (ts *Tribserver) CreateUser(
args *tribproto.CreateUserArgs, reply *tribproto.CreateUserReply) error {
var trib_key, fllw_key string
var err error
trib_key = fmt.Sprintf("%s:T", args.Userid)
fllw_key = fmt.Sprintf("%s:F", args.Userid)
_, err = ts.Store.GetList(trib_key)
if err == nil {
lsplog.Vlogf(0, "try create user %s , but exist !", args.Userid)
reply.Status = tribproto.EEXISTS
return nil
}
err = ts.Store.Put(trib_key, "")
if lsplog.CheckReport(2, err) {
lsplog.Vlogf(0, "user %s , trib_key exist !", args.Userid)
reply.Status = tribproto.EEXISTS
return nil
}
err = ts.Store.Put(fllw_key, "")
if lsplog.CheckReport(2, err) {
reply.Status = tribproto.EEXISTS
return nil
}
reply.Status = tribproto.OK
lsplog.Vlogf(0, "create user status %d", tribproto.OK)
return nil
}
/**@brief helper function for sorting
* @param server master storage server addr
* @param myhostport trib server's port
* @param flags
* @return *Libstore
* @return error
*/
func iNewLibstore(server, myhostport string, flags int) (*Libstore, error) {
var store Libstore
var master *rpc.Client
var args storageproto.GetServersArgs
var reply storageproto.RegisterReply
var err error
store.Addr = myhostport
store.Flags = flags
if store.Addr != "" {
rpc.Register(cacherpc.NewCacheRPC(&store))
}
lsplog.Vlogf(3, "libstore try to connect to master storage %s", server)
master, err = rpc.DialHTTP("tcp", server)
if lsplog.CheckReport(1, err) {
return nil, err
}
lsplog.Vlogf(3, "try to call GetServers")
master.Call("StorageRPC.GetServers", &args, &reply)
if !reply.Ready {
for i := 0; i < 5; i++ {
time.Sleep(1000 * time.Millisecond)
master.Call("StorageRPC.GetServers", &args, &reply)
}
}
err = master.Close()
if lsplog.CheckReport(1, err) {
lsplog.Vlogf(3, "WARNING close master failed")
}
// couldn't get list of servers from master
if (reply.Ready == false) || (reply.Servers == nil) {
return nil, lsplog.MakeErr("Storage system not ready.")
}
store.Nodes = reply.Servers
store.RPCConn = make([]*rpc.Client, len(store.Nodes))
sort.Sort(store.Nodes)
/*
for i := 0; i < len(store.Nodes); i++ {
fmt.Printf("%v\n", store.Nodes[i])
}*/
store.Leases = cache.NewCache()
if lsplog.CheckReport(1, err) {
return nil, err
}
lsplog.Vlogf(3, "libstore create complete")
return &store, nil
}
func (ss *Storageserver) CheckWithinRange(key string) bool {
if !ss.multiNodes {
lsplog.Vlogf(1, "[CheckWithinRange] Within range [Single node]")
return true
}
hash := getServerIndicator(key)
result := ss.getSuccessor(hash) == ss.nodeid
lsplog.Vlogf(1, "[CheckWithinRange] Key within range %v", result)
return result
}
func (ss *Storageserver) RemoveFromList(args *storageproto.PutArgs,
reply *storageproto.PutReply) error {
lsplog.Vlogf(0, "removeFromList key %s", args.Key)
//ss.rwlock.Lock()
val, present := ss.hash[args.Key]
if !present {
lsplog.Vlogf(3, "try to remove, key %s does not exist", args.Key)
reply.Status = storageproto.EKEYNOTFOUND
//ss.rwlock.Unlock()
return nil
}
entry, present := ss.leasePool[args.Key]
if present {
entry.mtx.Lock()
ss.revokeLeaseHolders(args.Key)
}
var list []string
err := json.Unmarshal([]byte(val), &list)
if err != nil {
lsplog.Vlogf(0, "WARNING: unmarshal data generate an error")
}
for i, v := range list {
if v == args.Value {
list = append(list[:i], list[i+1:]...)
ss.hash[args.Key], err = json.Marshal(list)
if err != nil {
lsplog.Vlogf(0, "WARNING: Marshal data generate an error")
}
reply.Status = storageproto.OK
//ss.rwlock.Unlock()
return nil
}
}
reply.Status = storageproto.EITEMNOTFOUND
if present {
entry.mtx.Unlock()
}
//ss.rwlock.Unlock()
return nil
}
func (ss *Storageserver) Get(args *storageproto.GetArgs, reply *storageproto.GetReply) error {
if !ss.CheckWithinRange(args.Key) {
reply.Status = storageproto.EWRONGSERVER
lsplog.Vlogf(1, "[Get] The key is given to the wrong server")
return nil
}
if args.WantLease {
ss.modifyingLock.Lock()
if ss.modifying[args.Key] != nil {
reply.Lease.Granted = false
} else {
reply.Lease.Granted = true
ss.leasesLock.Lock()
currentLeaseList := ss.leases[args.Key]
if currentLeaseList == nil {
currentLeaseList = list.New()
}
currentLeaseList.PushBack(&ClientLease{args.LeaseClient, time.Now().Add((storageproto.LEASE_SECONDS + storageproto.LEASE_GUARD_SECONDS) * time.Second)})
ss.leasesLock.Unlock()
}
ss.modifyingLock.Unlock()
}
ss.storageLock.Lock()
if ss.storage[args.Key] == nil {
reply.Status = storageproto.EKEYNOTFOUND
} else {
reply.Value = *ss.storage[args.Key]
reply.Status = storageproto.OK
}
ss.storageLock.Unlock()
return nil
}
// RPC-able interfaces, bridged via StorageRPC.
// These should do something! :-)
func (ss *Storageserver) Get(args *storageproto.GetArgs,
reply *storageproto.GetReply) error {
//ss.rwlock.RLock()
fmt.Printf("try to GET key %s\n", args.Key)
val, present := ss.hash[args.Key]
if !present {
//if the whole system only have one storage node
if ss.numnodes == 1 {
reply.Status = storageproto.EKEYNOTFOUND
} else {
//reply.Status = storageproto.EKEYNOTFOUND
reply.Status = storageproto.EWRONGSERVER
}
fmt.Printf("storage GET key %s failed, nonexist\n", args.Key)
//ss.rwlock.RUnlock()
return nil
}
err := json.Unmarshal([]byte(val), &(reply.Value))
if err != nil {
lsplog.Vlogf(0, "WARNING: unmarshal data generate an error")
}
if args.WantLease {
ss.addLeasePool(args, &(reply.Lease))
}
fmt.Printf("Storage Get key %s, val %s, lease %t\n",
args.Key, reply.Value, reply.Lease.Granted)
reply.Status = storageproto.OK
//ss.rwlock.RUnlock()
return nil
}
/**@brief Hashes a key and returns an RPC connection to the server
responsible for storing it. If an RPC connection is not
established, create one and store it for future accesses.
* @param server master server addr
* @param myhostport trib server's port
* @param flags
* @return *Libstore
* @return error
*/
func (ls *Libstore) GetServer(key string) (*rpc.Client, error) {
var id uint32
var svr int
var err error
//lsplog.Vlogf(3, "libstore GetServer Invoked")
id = Storehash(strings.Split(key, ":")[0])
// returns the index of the first server after the key's hash
svr = sort.Search(
len(ls.Nodes), func(i int) bool { return ls.Nodes[i].NodeID >= id })
svr = (svr) % len(ls.Nodes)
//lsplog.Vlogf(0, "%s -> %d (%d)\n", key, id, svr)
if ls.RPCConn[svr] == nil {
lsplog.Vlogf(0, "Caching RPC connection to %s.\n", ls.Nodes[svr].HostPort)
ls.RPCConn[svr], err = rpc.DialHTTP("tcp", ls.Nodes[svr].HostPort)
if lsplog.CheckReport(1, err) {
return nil, err
}
}
return ls.RPCConn[svr], nil
}
func main() {
lsplog.SetVerbose(10)
ls, e := libstore.NewLibstore("localhost:9009", "localhost:9999", 0)
e = ls.AppendToList("ammar:hey", "Helly")
e = ls.AppendToList("ammar:hey", "Helly")
e = ls.AppendToList("ammar:hey", "Kelly")
lsplog.Vlogf(1, "%v", e)
var k []string
k, e = ls.GetList("ammar:hey")
lsplog.Vlogf(1, "%v %v", k, e)
e = ls.RemoveFromList("ammar:hey", "Helly")
lsplog.Vlogf(1, "%v ", e)
e = ls.RemoveFromList("ammar:hey", "Helly")
lsplog.Vlogf(1, "%v ", e)
k, e = ls.GetList("ammar:hey")
lsplog.Vlogf(1, "%v %v", k, e)
}
func (ls *Libstore) getServerConnection(indicator uint32) (*rpc.Client, error) {
if len(ls.servers) == 1 {
return ls.connections[ls.servers[0].HostPort], nil
}
var smallest uint32
smallest = math.MaxUint32
smallNodeIndex := 0
nodeIndex := 0
for i := range ls.servers {
if smallest > ls.servers[i].NodeID {
smallest = ls.servers[i].NodeID
smallNodeIndex = i
}
}
var successor *uint32
for i := range ls.servers {
if indicator <= ls.servers[i].NodeID {
if successor == nil || ls.servers[i].NodeID < *successor {
successor = &ls.servers[i].NodeID
nodeIndex = i
lsplog.Vlogf(1, "[libstore] Updated to successor %v and index %v", ls.servers[i].NodeID, i)
}
}
}
if successor == nil {
nodeIndex = smallNodeIndex
successor = &smallest
}
hostport := ls.servers[nodeIndex].HostPort
lsplog.Vlogf(3, "[getServerConnection] Successfor NodeID is %v and its host:port is %s and index %v", *successor, hostport, nodeIndex)
if ls.connections[hostport] == nil {
lsplog.Vlogf(3, "[getServerConnection] Connection for %s is not created. Creating connection...", hostport)
con, e := rpc.DialHTTP("tcp", hostport)
if e != nil {
lsplog.Vlogf(3, "[getServerConnection] Connection for %s failed", hostport)
return nil, e
}
lsplog.Vlogf(3, "[getServerConnection] Connection created successfully")
ls.connections[hostport] = con
}
return ls.connections[hostport], nil
}
func (ss *Storageserver) Put(args *storageproto.PutArgs,
reply *storageproto.PutReply) error {
var err error
fmt.Printf("st svr put invoked key %s, val %s !!!\n", args.Key, args.Value)
//ss.rwlock.Lock()
if entry, present := ss.leasePool[args.Key]; present {
fmt.Printf("try to put to %s still lease pool, call revoke!!!\n", args.Key)
entry.mtx.Lock()
ss.revokeLeaseHolders(args.Key)
entry.mtx.Unlock()
}
_, present := ss.hash[args.Key]
if present {
ss.hash[args.Key], _ = json.Marshal(args.Value)
reply.Status = storageproto.OK
//ss.rwlock.Unlock()
return nil
}
if args.Value == "" {
lsplog.Vlogf(3, "storage first put %s", args.Key)
ss.hash[args.Key], err = json.Marshal([]string{})
} else {
//fmt.Printf("storage put %s, val %s", args.Key, args.Value)
ss.hash[args.Key], err = json.Marshal(args.Value)
}
if err != nil {
lsplog.Vlogf(0, "WARNING: Marshal data generate an error")
}
reply.Status = storageproto.OK
//ss.rwlock.Unlock()
//fmt.Println("storage put complete!")
return nil
}
func (ls *Libstore) iAppendToList(key, newitem string) error {
indicator := ls.getServerIndicator(key)
con, e := ls.getServerConnection(indicator)
if e != nil {
return e
}
args := &storageproto.PutArgs{key, newitem}
var reply storageproto.PutReply
lsplog.Vlogf(1, "Calling RPC StorageRPC.AppendToList with key %s and value %s", key, newitem)
e = con.Call("StorageRPC.AppendToList", args, &reply)
if e != nil {
return e
}
lsplog.Vlogf(1, "Called RPC StorageRPC.AppendToList successfully")
lsplog.Vlogf(1, "Status recieved %v", reply.Status)
if reply.Status == storageproto.EWRONGSERVER {
lsplog.Vlogf(1, "Key %s should not be called from server", key)
return lsplog.MakeErr("Wrong server is called")
} else if reply.Status == storageproto.EITEMNOTFOUND {
lsplog.Vlogf(1, "Item to add not found in server")
return lsplog.MakeErr("Error: item to add not found")
} else if reply.Status == storageproto.EITEMEXISTS {
lsplog.Vlogf(1, "Item to add alrady exists")
return lsplog.MakeErr("Error: item to add exists")
}
return nil
}
func (ls *Libstore) iPut(key, value string) error {
indicator := ls.getServerIndicator(key)
con, e := ls.getServerConnection(indicator)
if e != nil {
return e
}
args := &storageproto.PutArgs{key, value}
var reply storageproto.PutReply
lsplog.Vlogf(1, "Calling RPC StorageRPC.Put with key %s and value %s", key, value)
e = con.Call("StorageRPC.Put", args, &reply)
if e != nil {
return e
}
lsplog.Vlogf(1, "Called RPC StorageRPC.Put successfully")
lsplog.Vlogf(1, "Status recieved %v", reply.Status)
if reply.Status == storageproto.EWRONGSERVER {
lsplog.Vlogf(1, "Key %s should not be called from server", key)
return lsplog.MakeErr("Wrong server is called")
} else if reply.Status == storageproto.EPUTFAILED {
lsplog.Vlogf(1, "Error happened while putting")
return lsplog.MakeErr("Error while putting error")
} else if reply.Status != storageproto.OK {
lsplog.Vlogf(1, "Error happened while putting")
return lsplog.MakeErr("Error while putting error")
}
return nil
}
func (ss *Storageserver) getSuccessor(indicator uint32) uint32 {
smallest := ss.nodes[0].NodeID
for i := range ss.nodes {
if smallest > ss.nodes[i].NodeID {
smallest = ss.nodes[i].NodeID
}
}
lsplog.Vlogf(1, "Smallest nodeID is %v", smallest)
var successor *uint32
for i := range ss.nodes {
if indicator <= ss.nodes[i].NodeID {
if successor == nil || ss.nodes[i].NodeID < *successor {
lsplog.Vlogf(1, "[getSuccessor] Updated to successor %v", ss.nodes[i].NodeID)
successor = &ss.nodes[i].NodeID
}
}
}
if successor == nil {
successor = &smallest
}
lsplog.Vlogf(1, "[getSuccessor] Key belong nodeID is %v", smallest)
return *successor
}
func (ls *Libstore) getFromCache(key string) []string {
ls.cacheLocker.Lock()
if ls.cache[key] != nil {
leased := ls.cache[key]
if time.Now().Before(leased.expiryTime) {
ls.cacheLocker.Unlock()
lsplog.Vlogf(3, "[getFromCache] Getting a cached value %v", leased.value)
return leased.value
} else {
delete(ls.cache, key)
}
}
ls.cacheLocker.Unlock()
return nil
}
/**@brief create a new tribserver
* @param string
* @param string
* @return *tribserver
*/
func NewTribserver(storagemaster, myhostport string) *Tribserver {
lsplog.SetVerbose(3)
fmt.Printf("st_master:%s, port:%s\n", storagemaster, myhostport)
var svr *Tribserver = new(Tribserver)
var err error
lsplog.Vlogf(3, "try to create libstore")
// libstore.NONE forces no leases on Get and GetList requests
svr.Store, err =
libstore.NewLibstore(storagemaster, myhostport, libstore.NONE)
if lsplog.CheckReport(1, err) {
return nil
}
svr.Id = 1
return svr
}
func (ls *Libstore) iGet(key string) (string, error) {
cached := ls.getFromCache(key)
if cached != nil {
return cached[0], nil
}
indicator := ls.getServerIndicator(key)
con, e := ls.getServerConnection(indicator)
if e != nil {
return "", e
}
args := &storageproto.GetArgs{Key: key}
ls.checkWhetherLeaseNeeded(args, key)
var reply storageproto.GetReply
lsplog.Vlogf(1, "[Get] Calling RPC StorageRPC.Get on server")
e = con.Call("StorageRPC.Get", args, &reply)
if e != nil {
lsplog.Vlogf(1, "[Get] Calling RPC StorageRPC.Get has an error")
return "", e
}
lsplog.Vlogf(1, "[Get] Called RPC StorageRPC.Get successfully")
lsplog.Vlogf(1, "[Get] Status recieved %v", reply.Status)
if reply.Status == storageproto.EKEYNOTFOUND {
return "", lsplog.MakeErr("Key not found")
} else if reply.Status == storageproto.EWRONGSERVER {
lsplog.Vlogf(1, "[Get] Key %s should not be called from server", key)
return "", lsplog.MakeErr("Wrong server is called")
} else if reply.Status != storageproto.OK {
lsplog.Vlogf(1, "[Get] Error status %v is recieved from server", reply.Status)
return "", lsplog.MakeErr("Unknown error")
}
if reply.Lease.Granted {
if ls.flags == NONE {
ls.requestsLocker.Lock()
delete(ls.requests, key)
ls.requestsLocker.Unlock()
}
ls.cacheLocker.Lock()
lsplog.Vlogf(1, "[Get] Adding key %s to cache", key)
ls.cache[key] = &Leased{time.Now().Add(time.Duration(reply.Lease.ValidSeconds) * time.Second), []string{reply.Value}}
ls.cacheLocker.Unlock()
}
return reply.Value, nil
}
func (ss *Storageserver) revokeLeasesList(key string) {
ss.leasesListLock.Lock()
revokeList := ss.leasesList[key]
ss.leasesListLock.Unlock()
if revokeList == nil {
lsplog.Vlogf(1, "[revokeLeasesList] There are no leases to revoke")
return
}
for e := revokeList.Front(); e != nil; e = e.Next() {
c := (e.Value).(ClientLease)
if time.Now().After(c.expiryTime) {
continue
}
con := ss.getConnection(c.callbackAddress)
args := &storageproto.RevokeLeaseArgs{key}
var reply *storageproto.RevokeLeaseReply
con.Call("cacherpc.RevokeLease", args, reply)
}
ss.leasesListLock.Lock()
delete(ss.leasesList, key)
ss.leasesListLock.Unlock()
}
// Non-master servers to the master
func (ss *Storageserver) RegisterServer(args *storageproto.RegisterArgs,
reply *storageproto.RegisterReply) error {
fmt.Printf("st registerServer invoked\n")
if !ss.isMaster {
lsplog.Vlogf(0, "WARNING:Calling a non-master node to register")
return lsplog.MakeErr("Calling a non-master node to register")
}
_, present := ss.nodes[args.ServerInfo]
if !present {
//add to nodes
ss.nodes[args.ServerInfo] = true
fmt.Println("add nodes %v", args.ServerInfo)
}
fmt.Printf("master collect slave info %d/%d\n", len(ss.nodes), ss.numnodes)
reply.Servers = nil
if len(ss.nodes) == ss.numnodes {
reply.Ready = true
//ss.GetServers(nil, reply)
servers := make([]storageproto.Node, ss.numnodes)
i := 0
for node, _ := range ss.nodes {
//fmt.Printf("i: %d, info: %v\n", i, node)
servers[i] = node
i++
}
reply.Servers = servers
} else {
reply.Ready = false
}
return nil
}
func (ss *Storageserver) GetList(args *storageproto.GetArgs, reply *storageproto.GetListReply) error {
lsplog.Vlogf(1, "[GetList] GetList is called with Key=[%v]", args.Key)
if !ss.CheckWithinRange(args.Key) {
reply.Status = storageproto.EWRONGSERVER
lsplog.Vlogf(1, "[GetList] The key is given to the wrong server")
return nil
}
if args.WantLease {
lsplog.Vlogf(1, "[GetList] A lease is requested")
ss.modifyingListLock.Lock()
if ss.modifyingList[args.Key] != nil {
reply.Lease.Granted = false
} else {
reply.Lease.Granted = true
ss.leasesListLock.Lock()
currentLeaseList := ss.leasesList[args.Key]
if currentLeaseList == nil {
currentLeaseList = list.New()
}
currentLeaseList.PushBack(&ClientLease{args.LeaseClient, time.Now().Add((storageproto.LEASE_SECONDS + storageproto.LEASE_GUARD_SECONDS) * time.Second)})
ss.leasesListLock.Unlock()
}
ss.modifyingListLock.Unlock()
}
lsplog.Vlogf(1, "[GetList] Locking stoargeList")
ss.storageListLock.Lock()
if ss.storageList[args.Key] == nil {
reply.Status = storageproto.EKEYNOTFOUND
lsplog.Vlogf(1, "[GetList] returned EKEYNOTFOUND")
} else {
valueList := ss.storageList[args.Key]
reply.Value = make([]string, valueList.Len())
i := 0
for e := valueList.Front(); e != nil; e = e.Next() {
reply.Value[i] = e.Value.(string)
i++
}
reply.Status = storageproto.OK
lsplog.Vlogf(1, "[GetList] returned OK")
}
ss.storageListLock.Unlock()
return nil
}
// Non-master servers to the master
func (ss *Storageserver) RegisterServer(args *storageproto.RegisterArgs, reply *storageproto.RegisterReply) error {
lsplog.Vlogf(1, "[storageimpl] Got a registration call from [%+v]", args.ServerInfo)
ss.registeredLocker.Lock()
defer ss.registeredLocker.Unlock()
reply.Servers = make([]storageproto.Node, ss.numnodes)
if ss.registered == len(ss.nodes) {
lsplog.Vlogf(1, "[RegisterServer] Request is for getting ready")
reply.Ready = true
for i := range ss.nodes {
reply.Servers[i] = *ss.nodes[i]
}
return nil
}
lsplog.Vlogf(1, "[RegisterServer] Request is for registration")
for i := range ss.nodes {
if ss.nodes[i] != nil && ss.nodes[i].NodeID == args.ServerInfo.NodeID {
lsplog.Vlogf(1, "[RegisterServer] Already registered and not ready")
reply.Ready = false
return nil
}
}
lsplog.Vlogf(1, "[RegisterServer] Registering")
ss.nodes[ss.registered] = &args.ServerInfo
ss.registered++
if ss.registered == len(ss.nodes) {
lsplog.Vlogf(1, "[RegisterServer] Registered and it was the last node")
reply.Ready = true
for i := range ss.nodes {
reply.Servers[i] = *ss.nodes[i]
}
} else {
lsplog.Vlogf(1, "[RegisterServer] Registered and there are more nodes ")
reply.Ready = false
}
return nil
}
func (ss *Storageserver) AppendToList(args *storageproto.PutArgs, reply *storageproto.PutReply) error {
lsplog.Vlogf(1, "Recieved AppendToList request with key=[%v] item=[%v]", args.Key, args.Value)
if !ss.CheckWithinRange(args.Key) {
lsplog.Vlogf(1, "[AppendToList] Returning AppendToList with Status EWRONGSERVER")
reply.Status = storageproto.EWRONGSERVER
return nil
}
lsplog.Vlogf(1, "[AppendToList] Modifying list")
ss.modifyingListLock.Lock()
ss.modifyingList[args.Key] = new(bool)
ss.modifyingListLock.Unlock()
ss.revokeLeasesList(args.Key)
ss.storageListLock.Lock()
lsplog.Vlogf(1, "[AppendToList] Iterating list and looking for duplicates")
lst := ss.storageList[args.Key]
if lst == nil {
ss.storageList[args.Key] = list.New()
lst = ss.storageList[args.Key]
} else {
for e := lst.Front(); e != nil; e = e.Next() {
if e.Value == args.Value {
lsplog.Vlogf(1, "[AppendToList] Returning AppendToList with Status EITEMEXISTS")
reply.Status = storageproto.EITEMEXISTS
ss.storageListLock.Unlock()
return nil
}
}
}
lst.PushBack(args.Value)
ss.storageListLock.Unlock()
ss.modifyingListLock.Lock()
delete(ss.modifyingList, args.Key)
ss.modifyingListLock.Unlock()
reply.Status = storageproto.OK
lsplog.Vlogf(1, "[AppendToList] Returning AppendToList with Status OK")
return nil
}
func (ls *Libstore) revokeLeasesListenerInit(myhostport string) {
lsplog.Vlogf(1, "Creating cache callback in %s", myhostport)
cacheimpl := cacherpc.NewCacheRPC(ls)
rpc.Register(cacheimpl)
}
func (ls *Libstore) RevokeLease(args *storageproto.RevokeLeaseArgs, reply *storageproto.RevokeLeaseReply) error {
lsplog.Vlogf(1, "[RevokeLease] Revoking lease for %s", args.Key)
delete(ls.cache, args.Key)
reply.Status = storageproto.OK
return nil
}
func (ss *Storageserver) AppendToList(args *storageproto.PutArgs,
reply *storageproto.PutReply) error {
fmt.Printf("try append %s to list %s\n", args.Value, args.Key)
//ss.rwlock.Lock()
_, present := ss.hash[args.Key]
if !present {
ss.hash[args.Key] = nil
/*
fmt.Printf("try append %s list with %s, list not exist\n",args.Key, args.Value)
reply.Status = storageproto.EKEYNOTFOUND
ss.rwlock.Unlock()
return nil
*/
}
entry, present := ss.leasePool[args.Key]
if present {
//this mutex will ''queue'' later put request while revoking
entry.mtx.Lock()
ss.revokeLeaseHolders(args.Key)
}
fmt.Printf("storage append to %s list %s\n", args.Key, args.Value)
var list []string
err := json.Unmarshal([]byte(ss.hash[args.Key]), &list)
if err != nil {
lsplog.Vlogf(0, "WARNING: unmarshal data generate an error")
}
//need check duplicate before insertion
for _, v := range list {
if v == args.Value {
reply.Status = storageproto.EITEMEXISTS
//ss.rwlock.Unlock()
return nil
}
}
list = append(list, ([]string{args.Value})...)
ss.hash[args.Key], err = json.Marshal(list)
if err != nil {
lsplog.Vlogf(0, "WARNING: Marshal data generate an error")
}
reply.Status = storageproto.OK
fmt.Printf("comp apd %s to %s,val %s\n", args.Value, args.Key, ss.hash[args.Key])
//ss.rwlock.Unlock()
if present {
entry.mtx.Unlock()
}
return nil
}
func (ls *Libstore) getServerIndicator(key string) uint32 {
parts := strings.Split(key, ":")
hash := Storehash(parts[0])
lsplog.Vlogf(3, "[getServerIndicator] Key indicator %s key indicator hash %v", parts[0], hash)
return hash
}
func NewStorageserver(master string, numnodes int, portnum int, nodeid uint32) *Storageserver {
lsplog.Vlogf(1, "[storageimpl] master recieved %v numnodes %v", master, numnodes)
ss := new(Storageserver)
ss.storage = make(map[string]*string)
ss.storageLock = new(sync.Mutex)
ss.storageList = make(map[string]*list.List)
ss.storageListLock = new(sync.Mutex)
ss.leases = make(map[string]*list.List)
ss.leasesLock = new(sync.Mutex)
ss.portnum = portnum
ss.modifying = make(map[string]*bool)
ss.modifyingLock = new(sync.Mutex)
ss.numnodes = numnodes
ss.leasesList = make(map[string]*list.List)
ss.leasesListLock = new(sync.Mutex)
ss.modifyingList = make(map[string]*bool)
ss.modifyingListLock = new(sync.Mutex)
ss.callbackConnections = make(map[string]*rpc.Client)
ss.callbackLocker = new(sync.Mutex)
if nodeid == 0 {
reallySeedTheDamnRNG()
ss.nodeid = rand.Uint32()
} else {
ss.nodeid = nodeid
}
lsplog.Vlogf(1, "[NewStorageserver] This node's ID is %v", ss.nodeid)
if master == fmt.Sprintf("localhost:%v", portnum) {
ss.master = true
ss.nodes = make([]*storageproto.Node, ss.numnodes)
if ss.numnodes == 1 {
ss.multiNodes = false
lsplog.Vlogf(1, "[NewStorageserver] Running storage server on a single node")
} else {
ss.multiNodes = true
lsplog.Vlogf(1, "[NewStorageserver] Running storage server on multinodes")
}
ss.nodes[0] = &storageproto.Node{fmt.Sprintf("localhost:%v", ss.portnum), ss.nodeid}
ss.registered++
ss.registeredLocker = new(sync.Mutex)
lsplog.Vlogf(1, "[NewStorageserver] This node's info master=[%v] multiNodes=[%v] numnodes=[%v]", ss.master, ss.multiNodes, ss.numnodes)
} else {
ss.multiNodes = true
ss.master = false
hostport := fmt.Sprintf("%s", master)
args := &storageproto.RegisterArgs{storageproto.Node{fmt.Sprintf("localhost:%v", ss.portnum), ss.nodeid}}
lsplog.Vlogf(1, "[NewStorageserver] This node's info master=[%v] multiNodes=[%v] numnodes=[%v]", ss.master, ss.multiNodes, ss.numnodes)
var reply storageproto.RegisterReply
lsplog.Vlogf(1, "[NewStorageserver] Dialing master... %v", hostport)
cli, e := rpc.DialHTTP("tcp", hostport)
for e != nil {
lsplog.Vlogf(1, "[NewStorageserver] Dialing master failed. Attempting to dialing it again... %v", hostport)
time.Sleep(1 * time.Second)
cli, e = rpc.DialHTTP("tcp", hostport)
}
lsplog.Vlogf(1, "[NewStorageserver] Calling master... %v", hostport)
cli.Call("StorageRPC.Register", args, &reply)
for !reply.Ready {
lsplog.Vlogf(1, "[NewStorageserver] Calling master failed. Attempting to calling it again... %v", hostport)
time.Sleep(1 * time.Second)
cli.Call("StorageRPC.Register", args, &reply)
}
ss.nodes = make([]*storageproto.Node, len(reply.Servers))
for i := range reply.Servers {
ss.nodes[i] = &reply.Servers[i]
}
lsplog.Vlogf(1, "[NewStorageserver] Got following nodes %+v", ss.nodes)
}
return ss
}