// Broadcast sends a message to all peers with that have the hash in their keyspace.
func (s *Server) Broadcast(hash *uint64, msg *protocol.Message) error {
alreadySentTo := make(map[uint64]bool)
if msg.Gossip {
for _, to := range msg.SentTo {
alreadySentTo[to] = true
}
}
sentTo := []uint64{murmur3.Sum64([]byte(s.LocalPeer().Id))}
var toPeers []*Conn
for _, peer := range s.Peers {
peerHash := murmur3.Sum64([]byte(peer.Peer.Id))
if (hash == nil || peer.Peer.GetKeyspace().Includes(*hash)) && !alreadySentTo[peerHash] {
sentTo = append(sentTo, peerHash)
toPeers = append(toPeers, peer)
}
}
if msg.Gossip {
msg.SentTo = append(msg.SentTo, sentTo...)
}
for _, peer := range toPeers {
s.Printf("Broadcasting to %s", peer.Peer.Id)
if err := peer.Send(msg); err != nil {
return err
}
}
return nil
}
func (ds *InMemDS) Create(parent, id []byte, inode uint64, name string, attr *pb.Attr, isdir bool) (string, *pb.Attr, error) {
ds.Lock()
defer ds.Unlock()
p := murmur3.Sum64(parent)
if _, exists := ds.nodes[p].entries[name]; exists {
return "", &pb.Attr{}, nil
}
entry := &Entry{
path: name,
inode: inode,
isdir: isdir,
attr: attr,
xattrs: make(map[string][]byte),
blocks: 0,
}
if isdir {
entry.entries = make(map[string]uint64)
entry.ientries = make(map[uint64]string)
}
i := murmur3.Sum64(id)
ds.nodes[i] = entry
ds.nodes[p].entries[name] = i
ds.nodes[p].ientries[i] = name
atomic.AddUint64(&ds.nodes[p].nodeCount, 1)
return name, attr, nil
}
func (s *Store) getShard(key string) *_shard {
s.mutex.RLock()
num := int(murmur3.Sum64([]byte(key))>>1) % s.shardsCount
shard := s.shards[num]
s.mutex.RUnlock()
return shard
}
// LocalKeyspace returns the keyspace that the local node represents.
func (s *Server) LocalKeyspace() *protocol.Keyspace {
center := murmur3.Sum64([]byte(s.LocalID()))
return &protocol.Keyspace{
Start: center - math.MaxUint64/4,
End: center + math.MaxUint64/4,
}
}
func (ds *InMemDS) SetAttr(id []byte, attr *pb.Attr, v uint32) (*pb.Attr, error) {
ds.Lock()
defer ds.Unlock()
valid := fuse.SetattrValid(v)
if entry, ok := ds.nodes[murmur3.Sum64(id)]; ok {
if valid.Mode() {
entry.attr.Mode = attr.Mode
}
if valid.Size() {
if attr.Size == 0 {
entry.blocks = 0
entry.lastblock = 0
}
entry.attr.Size = attr.Size
}
if valid.Mtime() {
entry.attr.Mtime = attr.Mtime
}
if valid.Atime() {
entry.attr.Atime = attr.Atime
}
if valid.Uid() {
entry.attr.Uid = attr.Uid
}
if valid.Gid() {
entry.attr.Gid = attr.Gid
}
return entry.attr, nil
}
return &pb.Attr{}, nil
}
func NewInMemDS() *InMemDS {
ds := &InMemDS{
nodes: make(map[uint64]*Entry),
}
n := &Entry{
path: "/",
inode: 1,
isdir: true,
entries: make(map[string]uint64),
ientries: make(map[uint64]string),
}
ts := time.Now().Unix()
n.attr = &pb.Attr{
Inode: n.inode,
Atime: ts,
Mtime: ts,
Ctime: ts,
Crtime: ts,
Mode: uint32(os.ModeDir | 0775),
Uid: 1001, // TODO: need to config default user/group id
Gid: 1001,
}
ds.nodes[murmur3.Sum64(GetID([]byte("1"), n.attr.Inode, 0))] = n
return ds
}
// signAndInsertTriples signs a set of triples with the server's key and then inserts them into the graph.
func (s *server) signAndInsertTriples(triples []*protocol.Triple, key *crypto.PrivateKey) error {
hashes := make(map[uint64][]*protocol.Triple)
unix := time.Now().Unix()
for _, triple := range triples {
if err := key.SignTriple(triple); err != nil {
return err
}
triple.Created = unix
hash := murmur3.Sum64([]byte(triple.Subj))
hashes[hash] = append(hashes[hash], triple)
}
for hash, triples := range hashes {
msg := &protocol.Message{
Message: &protocol.Message_InsertTriples{
InsertTriples: &protocol.InsertTriples{
Triples: triples,
}},
Gossip: true,
}
currentKeyspace := s.network.LocalPeer().Keyspace.Includes(hash)
if err := s.network.Broadcast(&hash, msg); currentKeyspace && err == network.ErrNoRecipients {
} else if err != nil {
return err
}
if currentKeyspace {
s.ts.Insert(triples)
}
}
return nil
}
func verifyShardDist(t *testing.T, client *ConsistentHashRes, nShard int, n int) {
sdMax := float64(6)
result := make(map[string]int)
fmt.Println("========================")
for i := 0; i < n; i++ {
id := murmur3.Sum64(uuid.NewV4().Bytes())
info, ok := client.Get(fmt.Sprintf("%d", id))
assert.True(t, ok, "should get shard info")
if _, b := result[info]; b == true {
result[info]++
} else {
result[info] = 1
}
}
avg := float64(100.0) / float64(nShard)
var sum float64
for key, val := range result {
fmt.Printf("%s, count = %d\n", key, val)
sum += math.Pow(((100.0 * float64(val) / float64(n)) - avg), 2)
}
sd := math.Sqrt(sum / float64(nShard))
fmt.Printf("average: %.3f%% standard deviation: %.3f%%\n", avg, sd)
if sd > sdMax {
assert.Fail(t, fmt.Sprintf("standard deviation is too high %v", sd))
}
}
func (ds *InMemDS) GetAttr(id []byte) (*pb.Attr, error) {
ds.RLock()
defer ds.RUnlock()
if entry, ok := ds.nodes[murmur3.Sum64(id)]; ok {
return entry.attr, nil
}
return &pb.Attr{}, nil
}
func (ds *InMemDS) Getxattr(id []byte, name string) (*pb.GetxattrResponse, error) {
ds.RLock()
defer ds.RUnlock()
if xattr, ok := ds.nodes[murmur3.Sum64(id)].xattrs[name]; ok {
return &pb.GetxattrResponse{Xattr: xattr}, nil
}
return &pb.GetxattrResponse{}, nil
}
func (ds *InMemDS) Removexattr(id []byte, name string) (*pb.RemovexattrResponse, error) {
ds.Lock()
defer ds.Unlock()
if entry, ok := ds.nodes[murmur3.Sum64(id)]; ok {
delete(entry.xattrs, name)
}
return &pb.RemovexattrResponse{}, nil
}
func (ds *InMemDS) Setxattr(id []byte, name string, value []byte) (*pb.SetxattrResponse, error) {
ds.Lock()
defer ds.Unlock()
if entry, ok := ds.nodes[murmur3.Sum64(id)]; ok {
entry.xattrs[name] = value
}
return &pb.SetxattrResponse{}, nil
}
// subjInKeyspace appends an increasing number to the prefix until it finds a
// string that will hash into the given keyspace.
func subjInKeyspace(keyspace *protocol.Keyspace, prefix string) string {
subj := prefix
i := 1
for !keyspace.Includes(murmur3.Sum64([]byte(subj))) {
subj = prefix + strconv.Itoa(i)
i++
}
return subj
}
func (ds *InMemDS) Rename(oldParent, newParent []byte, oldName, newName string) (*pb.RenameResponse, error) {
ds.Lock()
defer ds.Unlock()
p := murmur3.Sum64(oldParent)
if id, ok := ds.nodes[p].entries[oldName]; ok {
// remove old
delete(ds.nodes[p].entries, oldName)
delete(ds.nodes[p].ientries, id)
atomic.AddUint64(&ds.nodes[p].nodeCount, ^uint64(0)) // -1
// add new
ds.nodes[id].path = newName
np := murmur3.Sum64(newParent)
ds.nodes[np].entries[newName] = id
ds.nodes[np].ientries[id] = newName
atomic.AddUint64(&ds.nodes[np].nodeCount, 1)
}
return &pb.RenameResponse{}, nil
}
func (ds *InMemDS) Lookup(parent []byte, name string) (string, *pb.Attr, error) {
ds.RLock()
defer ds.RUnlock()
id, ok := ds.nodes[murmur3.Sum64(parent)].entries[name]
if !ok {
return "", &pb.Attr{}, nil
}
entry := ds.nodes[id]
return entry.path, entry.attr, nil
}
func (s *Server) LocalPeer() *protocol.Peer {
id := fmt.Sprintf("%s:%d", s.IP, s.Port)
center := murmur3.Sum64([]byte(id))
keyspace := &protocol.Keyspace{
Start: center - math.MaxUint64/4,
End: center + math.MaxUint64/4,
}
return &protocol.Peer{
Id: id,
Keyspace: keyspace,
}
}
func main() {
nodeMap = make(map[uint64]string)
FirstNode := "http://localhost:3000/"
SecondNode := "http://localhost:3001/"
ThirdNode := "http://localhost:3002/"
keys = append(keys, murmur3.Sum64([]byte(FirstNode)))
keys = append(keys, murmur3.Sum64([]byte(SecondNode)))
keys = append(keys, murmur3.Sum64([]byte(ThirdNode)))
keys.Sort()
fmt.Println("Keys array is : ", keys)
for _, element := range keys {
switch element {
case murmur3.Sum64([]byte(FirstNode)):
nodeMap[element] = FirstNode
case murmur3.Sum64([]byte(SecondNode)):
nodeMap[element] = SecondNode
case murmur3.Sum64([]byte(ThirdNode)):
nodeMap[element] = ThirdNode
}
}
mux := routes.New()
mux.Put("/keys/:keyID/:value", PutValue)
mux.Get("/keys/:keyID", RetrieveValue)
http.Handle("/", mux)
http.ListenAndServe(":8080", nil)
}
func (self *Node) Get(key *string, data *[]byte) error {
var found bool
self.RLock()
if *data, found = self.data[*key]; found {
self.RUnlock()
return nil
}
self.RUnlock()
closestpeer := self.Peers.ClosestPeer(murmur3.Sum64([]byte(*key)) >> 16)
log.Println(self.Key(), closestpeer.Key())
var err error
*data, err = self.Peers.Get(closestpeer, *key)
return err
}
func (ds *InMemDS) Remove(parent []byte, name string) (int32, error) {
ds.Lock()
defer ds.Unlock()
p := murmur3.Sum64(parent)
id, ok := ds.nodes[p].entries[name]
if !ok {
return 1, nil
}
delete(ds.nodes, id)
delete(ds.nodes[p].entries, name)
delete(ds.nodes[p].ientries, id)
atomic.AddUint64(&ds.nodes[p].nodeCount, ^uint64(0)) // -1
return 0, nil
}
func (ds *InMemDS) Listxattr(id []byte) (*pb.ListxattrResponse, error) {
ds.RLock()
defer ds.RUnlock()
resp := &pb.ListxattrResponse{}
if entry, ok := ds.nodes[murmur3.Sum64(id)]; ok {
names := ""
for name := range entry.xattrs {
names += name
names += "\x00"
}
resp.Xattr = []byte(names)
}
return resp, nil
}
func (ds *InMemDS) Symlink(parent, id []byte, name string, target string, attr *pb.Attr, inode uint64) (*pb.SymlinkResponse, error) {
ds.Lock()
defer ds.Unlock()
p := murmur3.Sum64(parent)
if _, exists := ds.nodes[p].entries[name]; exists {
return &pb.SymlinkResponse{}, nil
}
entry := &Entry{
path: name,
inode: inode,
isdir: false,
islink: true,
target: target,
attr: attr,
xattrs: make(map[string][]byte),
}
i := murmur3.Sum64(id)
ds.nodes[i] = entry
ds.nodes[p].entries[name] = i
ds.nodes[p].ientries[i] = name
atomic.AddUint64(&ds.nodes[p].nodeCount, 1)
return &pb.SymlinkResponse{Name: name, Attr: attr}, nil
}
func getNode(key string) string {
keyHash := murmur3.Sum64([]byte(key))
var returnIndex = len(keys) - 1
for index, element := range keys {
if keyHash < element {
if index > 0 {
returnIndex = index - 1
}
break
}
}
return nodeMap[keys[returnIndex]]
}
func (ds *InMemDS) ReadDirAll(id []byte) (*pb.ReadDirAllResponse, error) {
ds.RLock()
defer ds.RUnlock()
e := &pb.ReadDirAllResponse{}
for i, _ := range ds.nodes[murmur3.Sum64(id)].ientries {
entry := ds.nodes[i]
if entry.isdir {
e.DirEntries = append(e.DirEntries, &pb.DirEnt{Name: entry.path, Attr: entry.attr})
} else {
e.FileEntries = append(e.FileEntries, &pb.DirEnt{Name: entry.path, Attr: entry.attr})
}
}
sort.Sort(ByDirent(e.DirEntries))
sort.Sort(ByDirent(e.FileEntries))
return e, nil
}
func (ds *InMemDS) Update(id []byte, block, blocksize, size uint64, mtime int64) {
// NOTE: Not sure what this function really should look like yet
i := murmur3.Sum64(id)
blocks := ds.nodes[i].blocks
if block >= blocks {
ds.nodes[i].blocks = block + 1
ds.nodes[i].lastblock = size
ds.nodes[i].blocksize = blocksize
ds.nodes[i].attr.Size = blocksize*block + size
} else if block == (blocks - 1) {
ds.nodes[i].lastblock = size
ds.nodes[i].attr.Size = blocksize*block + size
}
ds.nodes[i].attr.Mtime = mtime
}
// handleInsertTriple inserts set of triples into the graph.
func (s *server) handleInsertTriple(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "endpoint needs POST", 400)
return
}
body, _ := ioutil.ReadAll(r.Body)
var triples []*protocol.Triple
if err := json.Unmarshal(body, &triples); err != nil {
http.Error(w, err.Error(), 500)
return
}
hashes := make(map[uint64][]*protocol.Triple)
unix := time.Now().Unix()
for _, triple := range triples {
// TODO(d4l3k): This should ideally be refactored and force the client to presign the triple.
if err := s.crypto.SignTriple(triple); err != nil {
http.Error(w, err.Error(), 500)
return
}
triple.Created = unix
hash := murmur3.Sum64([]byte(triple.Subj))
hashes[hash] = append(hashes[hash], triple)
}
for hash, triples := range hashes {
msg := &protocol.Message{
Message: &protocol.Message_InsertTriples{
InsertTriples: &protocol.InsertTriples{
Triples: triples,
}},
Gossip: true,
}
currentKeyspace := s.network.LocalPeer().Keyspace.Includes(hash)
if err := s.network.Broadcast(&hash, msg); currentKeyspace && err == network.ErrNoRecipients {
} else if err != nil {
http.Error(w, err.Error(), 400)
return
}
if currentKeyspace {
s.ts.Insert(triples)
}
}
w.Write([]byte(fmt.Sprintf("Inserted %d triples.", len(triples))))
}
func (self *Node) AllocateTask(task *tasks.Task, result *[]byte) error {
self.routelock.Lock()
defer self.routelock.Unlock()
if task.Id == 0 {
task.Id = murmur3.Sum64([]byte(fmt.Sprintf("%v", time.Now().UnixNano()))) >> 16
}
//temp := time.Now()
//self.timeQueue(temp, string(task.Id))
task.Add(self.Addr)
for true {
if self.Compute && (int(atomic.LoadInt64(&self.TaskValue)+int64(task.Value)) < 10000 || len(task.Jumps) >= 48) {
//Updates current processing value to reflect task queue
atomic.AddInt64(&self.TaskValue, int64(task.Value))
//Processes Task
data, err := self.process(task)
//Updates current processing value to reflect task queue
atomic.AddInt64(&self.TaskValue, int64(-task.Value))
*result = data
if err == nil {
log.Println("Successful Process")
return err
}
//self.incrementTotalTaskFailures(string(task.Id))
log.Println(err)
} else {
var err error
peer := self.Peers.GetAPeer()
tries := 0
for task.Visited(peer.Addr) && tries < 48 {
time.Sleep(1 * time.Millisecond)
peer = self.Peers.GetAPeer()
tries++
}
*result, err = self.Peers.AllocateTask(peer, task)
if err == nil {
//self.incrementTotalRoutedTasks(string(task.Id))
return nil
}
log.Println(err)
}
}
*result = []byte("Error")
return nil
}
// handleInsertTriple inserts a triple into the graph.
func (s *server) handleInsertTriple(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "endpoint needs POST", 400)
return
}
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), 400)
return
}
subj := r.FormValue("subj")
pred := r.FormValue("pred")
obj := r.FormValue("obj")
lang := r.FormValue("lang")
triple := &protocol.Triple{
Subj: subj,
Pred: pred,
Obj: obj,
Lang: lang,
}
// TODO(d4l3k): This should ideally be refactored and force the client to presign the triple.
if err := s.crypto.SignTriple(triple); err != nil {
http.Error(w, err.Error(), 500)
return
}
msg := &protocol.Message{
Message: &protocol.Message_InsertTriples{
InsertTriples: &protocol.InsertTriples{
Triples: []*protocol.Triple{triple},
},
},
Gossip: true,
}
hash := murmur3.Sum64([]byte(triple.Subj))
if err := s.network.Broadcast(&hash, msg); err != nil {
http.Error(w, err.Error(), 400)
return
}
if s.network.LocalPeer().Keyspace.Includes(hash) {
s.ts.Insert(msg.GetInsertTriples().Triples)
}
http.Redirect(w, r, "/static/insert.html", 307)
}
func ShardQueryByHash(step *protocol.ArrayOp) map[uint64]*protocol.ArrayOp {
m := make(map[uint64]*protocol.ArrayOp)
// TODO(d4l3k): better query hash splitting
var bad bool
if len(step.Triples) > 0 {
for _, triple := range step.Triples {
if len(triple.Subj) == 0 {
bad = true
break
}
hash := murmur3.Sum64([]byte(triple.Subj))
m[hash] = step
}
} else {
bad = true
}
if bad {
return map[uint64]*protocol.ArrayOp{0: step}
}
return m
}
func (s *server) handleInsertTriples(conn *network.Conn, msg *protocol.Message) {
triples := msg.GetInsertTriples().Triples
localKS := s.network.LocalPeer().Keyspace
var validTriples []*protocol.Triple
idHashes := make(map[string]uint64)
for _, triple := range triples {
hash, ok := idHashes[triple.Subj]
if !ok {
hash = murmur3.Sum64([]byte(triple.Subj))
idHashes[triple.Subj] = hash
}
if !localKS.Includes(hash) {
s.Printf("ERR insert triple dropped due to keyspace %#v from %#v", triple, conn.Peer)
// TODO(d4l3k): Follow up on bad triple by reannouncing keyspace.
continue
}
validTriples = append(validTriples, triple)
}
s.ts.Insert(validTriples)
}
// Bloom returns a ScalableBloomFilter containing all the triples the current node has in the optional keyspace.
func (ts *TripleStore) Bloom(keyspace *protocol.Keyspace) (*boom.ScalableBloomFilter, error) {
filter := boom.NewDefaultScalableBloomFilter(BloomFalsePositiveRate)
results, errs := ts.EachTripleBatch(DefaultTripleBatchSize)
for triples := range results {
for _, triple := range triples {
if keyspace != nil {
hash := murmur3.Sum64([]byte(triple.Subj))
if !keyspace.Includes(hash) {
continue
}
}
data, err := triple.Marshal()
if err != nil {
return nil, err
}
filter.Add(data)
}
}
for err := range errs {
return nil, err
}
return filter, nil
}